diff --git a/.changeset/add-node-engine.md b/.changeset/add-node-engine.md new file mode 100644 index 000000000..10744b77a --- /dev/null +++ b/.changeset/add-node-engine.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +feat(v4)!: Add Node Engine to package json diff --git a/.changeset/remove-eve-pre-034.md b/.changeset/remove-eve-pre-034.md new file mode 100644 index 000000000..1e3034506 --- /dev/null +++ b/.changeset/remove-eve-pre-034.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(eve)!: Bump minimum eve version to `0.34.0` diff --git a/.changeset/remove-flue-pre-v1.md b/.changeset/remove-flue-pre-v1.md new file mode 100644 index 000000000..a029265f8 --- /dev/null +++ b/.changeset/remove-flue-pre-v1.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(v4)!: Remove flue pre-v1 support diff --git a/.changeset/remove-js-braintrust-cli.md b/.changeset/remove-js-braintrust-cli.md new file mode 100644 index 000000000..9dfca85f2 --- /dev/null +++ b/.changeset/remove-js-braintrust-cli.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref(v4)!: Remove JS Braintrust CLI diff --git a/.changeset/remove-plugins-public-api.md b/.changeset/remove-plugins-public-api.md new file mode 100644 index 000000000..21275cc2e --- /dev/null +++ b/.changeset/remove-plugins-public-api.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Remove plugins as part of public API diff --git a/.changeset/remove-zod-derived-types.md b/.changeset/remove-zod-derived-types.md new file mode 100644 index 000000000..c741bede7 --- /dev/null +++ b/.changeset/remove-zod-derived-types.md @@ -0,0 +1,5 @@ +--- +"braintrust": major +--- + +ref!: Remove Zod derived types from public SDK declarations diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 74b97d78a..4b4d6ce8c 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -236,11 +236,10 @@ jobs: pnpm install --frozen-lockfile || { echo "Baseline install failed"; git checkout "$CURRENT_REF"; exit 0; } pnpm run build || { echo "Baseline build failed"; git checkout "$CURRENT_REF"; exit 0; } - mkdir -p "$BASELINE_DIR/js/dev" "$BASELINE_DIR/js/util" + mkdir -p "$BASELINE_DIR/js/util" cp -R js/dist "$BASELINE_DIR/js/" - cp -R js/dev/dist "$BASELINE_DIR/js/dev/" cp -R js/util/dist "$BASELINE_DIR/js/util/" - echo "Copied js/dist, js/dev/dist, and js/util/dist to baseline" + echo "Copied js/dist and js/util/dist to baseline" git checkout "$CURRENT_REF" @@ -473,17 +472,6 @@ jobs: - name: Run hermetic tests working-directory: ./js run: pnpm run test:checks - - name: Run zod v3 tests - if: matrix.zod-version == '3.25.34' - working-directory: ./js - run: | - pnpm test:zod-v3 - - name: Run zod v4 tests - if: matrix.zod-version == '4.2.1' - working-directory: ./js - run: | - pnpm test:zod-v4 - checks-passed: name: checks-passed needs: diff --git a/AGENTS.md b/AGENTS.md index 6338e8830..1b62e3529 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,21 @@ mise install # Install toolchain and dependencies pnpm run build # Build all workspace packages (from repo root) ``` +## Public TypeScript APIs + +Do not derive SDK-owned public TypeScript types from Zod schemas (for example, +with `z.infer`, `z.input`, `z.output`, or equivalent schema-derived aliases). +Define public API types explicitly with interfaces, type aliases, or generated +plain types. Generic APIs may still infer types from caller-provided schemas. +When exporting a runtime validator, give it a compact public type such as +`z.ZodType` and test that the validator and public type stay in sync. + +Zod-derived public declarations can expand into large schema implementation +graphs. Those declarations are expensive for downstream TypeScript consumers to +parse, instantiate, and type-check, increasing compile time, declaration size, +and memory usage. They also expose validation-library implementation details as +part of the SDK's API surface. + ## Instrumentation Use the normal Orchestrion config plus plugin/channel path by default. Special-case source patches should be rare exceptions only when the target SDK cannot be instrumented through the standard transformer path, and the reason should be documented next to the patch. diff --git a/README.md b/README.md index d8fd417e4..d9858e7cb 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Eval("Say Hi Bot", { Run it: ```bash -BRAINTRUST_API_KEY= npx braintrust eval tutorial.eval.ts +BRAINTRUST_API_KEY= npx bt eval tutorial.eval.ts ``` ## Packages diff --git a/e2e/config/pr-comment-scenarios.json b/e2e/config/pr-comment-scenarios.json index eb2b3f5f4..6db8e7692 100644 --- a/e2e/config/pr-comment-scenarios.json +++ b/e2e/config/pr-comment-scenarios.json @@ -600,14 +600,6 @@ "label": "Flue Instrumentation", "metadataScenario": "flue-instrumentation", "variants": [ - { - "variantKey": "flue-v0-8-0", - "label": "v0.8 pinned" - }, - { - "variantKey": "flue-v0-8-latest", - "label": "v0.8 latest" - }, { "variantKey": "flue-v1-0-0-beta-3", "label": "v1 pinned" @@ -631,7 +623,7 @@ "label": "Eve Instrumentation", "metadataScenario": "eve-instrumentation", "variants": [ - { "variantKey": "eve-v0", "label": "v0 pinned" }, + { "variantKey": "eve-v0", "label": "v0.34 pinned" }, { "variantKey": "eve-v0-latest", "label": "v0 latest" } ] }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json index f9869502c..c3eedeee9 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.json @@ -53,6 +53,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -233,8 +235,6 @@ "output": "Researcher result: [Eve instrumentation](https://eve.dev/docs/guides/instrumentation)", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -365,6 +365,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -525,6 +527,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -705,8 +709,6 @@ "output": "Researcher result: Eve instrumentation https://eve.dev/docs/guides/instrumentation", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -912,6 +914,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt index 1006edff0..feec9910f 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0-latest.span-tree.txt @@ -65,6 +65,8 @@ span_tree: │ │ ] │ │ metadata: { │ │ "eve.session_id": "", +│ │ "model": "qwen/qwen3-30b-a3b", +│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -95,8 +97,6 @@ span_tree: │ │ output: "Researcher result: [Eve instrumentation](https://eve.dev/docs/guides/instrumentation)" │ │ metadata: { │ │ "eve.session_id": "", -│ │ "model": "qwen/qwen3-30b-a3b", -│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -347,6 +347,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", +│ "model": "qwen/qwen3-30b-a3b", +│ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -498,6 +500,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", + │ "model": "qwen/qwen3-30b-a3b", + │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -528,8 +532,6 @@ span_tree: │ output: "Researcher result: Eve instrumentation https://eve.dev/docs/guides/instrumentation" │ metadata: { │ "eve.session_id": "", - │ "model": "qwen/qwen3-30b-a3b", - │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -855,6 +857,8 @@ span_tree: ] metadata: { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" } diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json index 6066e92ae..af1d20932 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.json @@ -53,6 +53,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -233,8 +235,6 @@ "output": "Researcher result: {\"title\": \"Eve instrumentation\", \"url\": \"https://eve.dev/docs/guides/instrumentation\"}", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -365,6 +365,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -525,6 +527,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -705,8 +709,6 @@ "output": "Researcher result: Eve instrumentation (https://eve.dev/docs/guides/instrumentation)", "metadata": { "eve.session_id": "", - "model": "qwen/qwen3-30b-a3b", - "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, @@ -912,6 +914,8 @@ ], "metadata": { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" }, diff --git a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt index d86328682..1a17de807 100644 --- a/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt +++ b/e2e/scenarios/eve-instrumentation/__snapshots__/eve-v0.span-tree.txt @@ -65,6 +65,8 @@ span_tree: │ │ ] │ │ metadata: { │ │ "eve.session_id": "", +│ │ "model": "qwen/qwen3-30b-a3b", +│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -95,8 +97,6 @@ span_tree: │ │ output: "Researcher result: {\"title\": \"Eve instrumentation\", \"url\": \"https://eve.dev/docs/guides/instrumentation\"}" │ │ metadata: { │ │ "eve.session_id": "", -│ │ "model": "qwen/qwen3-30b-a3b", -│ │ "provider": "openrouter", │ │ "scenario": "eve-instrumentation", │ │ "testRunId": "" │ │ } @@ -347,6 +347,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", +│ "model": "qwen/qwen3-30b-a3b", +│ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -498,6 +500,8 @@ span_tree: │ ] │ metadata: { │ "eve.session_id": "", + │ "model": "qwen/qwen3-30b-a3b", + │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -528,8 +532,6 @@ span_tree: │ output: "Researcher result: Eve instrumentation (https://eve.dev/docs/guides/instrumentation)" │ metadata: { │ "eve.session_id": "", - │ "model": "qwen/qwen3-30b-a3b", - │ "provider": "openrouter", │ "scenario": "eve-instrumentation", │ "testRunId": "" │ } @@ -855,6 +857,8 @@ span_tree: ] metadata: { "eve.session_id": "", + "model": "qwen/qwen3-30b-a3b", + "provider": "openrouter", "scenario": "eve-instrumentation", "testRunId": "" } diff --git a/e2e/scenarios/eve-instrumentation/agent/agent.ts b/e2e/scenarios/eve-instrumentation/agent/agent.ts index 589c856cc..8b518295b 100644 --- a/e2e/scenarios/eve-instrumentation/agent/agent.ts +++ b/e2e/scenarios/eve-instrumentation/agent/agent.ts @@ -18,11 +18,15 @@ const dynamicModel = withReadableReasoning( ); export default defineAgent({ + experimental: { + instrumentationProviders: true, + }, model: defineDynamic({ - fallback: dynamicModel, events: { - "step.started": () => dynamicModel, + "step.started": () => ({ + model: dynamicModel, + modelContextWindowTokens: 8_192, + }), }, }), - modelContextWindowTokens: 8_192, }); diff --git a/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts b/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts new file mode 100644 index 000000000..b74a16488 --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/agent/channels/eve.ts @@ -0,0 +1,7 @@ +import { none } from "eve/channels/auth"; +import { eveChannel } from "eve/channels/eve"; + +export default eveChannel({ + // This local e2e fixture has no user accounts and only serves the test runner. + auth: none(), +}); diff --git a/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts deleted file mode 100644 index fcac75fde..000000000 --- a/e2e/scenarios/eve-instrumentation/agent/hooks/braintrust.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { braintrustEveHook } from "braintrust"; -import { defineState } from "eve/context"; -import { defineHook } from "eve/hooks"; - -export default defineHook( - braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-instrumentation", - ...(process.env.BRAINTRUST_E2E_RUN_ID - ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } - : {}), - }, - }), -); diff --git a/e2e/scenarios/eve-instrumentation/agent/instrumentation.ts b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts similarity index 66% rename from e2e/scenarios/eve-instrumentation/agent/instrumentation.ts rename to e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts index fff8a737d..9ba5484f2 100644 --- a/e2e/scenarios/eve-instrumentation/agent/instrumentation.ts +++ b/e2e/scenarios/eve-instrumentation/agent/instrumentation/braintrust.ts @@ -1,10 +1,14 @@ import { braintrustEveInstrumentation, initLogger } from "braintrust"; -import { defineState } from "eve/context"; import { defineInstrumentation } from "eve/instrumentation"; export default defineInstrumentation( braintrustEveInstrumentation({ - defineState, + metadata: { + scenario: "eve-instrumentation", + ...(process.env.BRAINTRUST_E2E_RUN_ID + ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } + : {}), + }, setup: ({ agentName }) => { initLogger({ projectName: process.env.BRAINTRUST_E2E_PROJECT_NAME || agentName, diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts index 955d05a88..158110861 100644 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/agent.ts @@ -11,6 +11,9 @@ const openrouter = createOpenRouter({ export default defineAgent({ description: "Research the Eve instrumentation documentation before the parent reads it.", + experimental: { + instrumentationProviders: true, + }, model: withReadableReasoning( openrouter("qwen/qwen3-30b-a3b", { provider: { diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts deleted file mode 100644 index fcac75fde..000000000 --- a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/hooks/braintrust.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { braintrustEveHook } from "braintrust"; -import { defineState } from "eve/context"; -import { defineHook } from "eve/hooks"; - -export default defineHook( - braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-instrumentation", - ...(process.env.BRAINTRUST_E2E_RUN_ID - ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } - : {}), - }, - }), -); diff --git a/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts new file mode 100644 index 000000000..9ba5484f2 --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/agent/subagents/researcher/instrumentation/braintrust.ts @@ -0,0 +1,18 @@ +import { braintrustEveInstrumentation, initLogger } from "braintrust"; +import { defineInstrumentation } from "eve/instrumentation"; + +export default defineInstrumentation( + braintrustEveInstrumentation({ + metadata: { + scenario: "eve-instrumentation", + ...(process.env.BRAINTRUST_E2E_RUN_ID + ? { testRunId: process.env.BRAINTRUST_E2E_RUN_ID } + : {}), + }, + setup: ({ agentName }) => { + initLogger({ + projectName: process.env.BRAINTRUST_E2E_PROJECT_NAME || agentName, + }); + }, + }), +); diff --git a/e2e/scenarios/eve-instrumentation/cassette-filter.mjs b/e2e/scenarios/eve-instrumentation/cassette-filter.mjs index f73d1a5ef..4066cef31 100644 --- a/e2e/scenarios/eve-instrumentation/cassette-filter.mjs +++ b/e2e/scenarios/eve-instrumentation/cassette-filter.mjs @@ -1,5 +1,117 @@ // @ts-check -export { filter } from "../ai-sdk-instrumentation/cassette-filter.mjs"; +const EVE_BUILTIN_TOOLS = new Set([ + "agent", + "ask_question", + "bash", + "glob", + "grep", + "load_skill", + "read_file", + "todo", + "web_fetch", + "write_file", +]); + +/** @type {import("@braintrust/seinfeld").FilterSpec} */ +export const filter = [ + "default", + { + ignoreBodyFields: [ + "experimental_generateMessageId", + "messageId", + "messages.*.id", + "messages.*.experimental_messageId", + // Eve changed the runtime-owned subagent output-schema declaration + // without changing this fixture's authored tool contract. + "tools.*.function.parameters.properties.outputSchema", + // JSON Schema's draft marker is metadata, not part of the accepted input. + "tools.*.function.parameters.$schema", + ], + normalizeRequest(request) { + if ( + request.body.kind !== "json" || + request.body.value === null || + typeof request.body.value !== "object" || + Array.isArray(request.body.value) + ) { + return request; + } + const body = request.body.value; + if (!Array.isArray(body.tools)) { + return request; + } + const messages = Array.isArray(body.messages) + ? body.messages.map((message) => message) + : body.messages; + if (Array.isArray(messages)) { + for (let index = 0; index < messages.length; index++) { + const message = messages[index]; + if ( + message === null || + typeof message !== "object" || + !("role" in message) || + message.role !== "tool" + ) { + continue; + } + let end = index + 1; + while ( + end < messages.length && + messages[end] !== null && + typeof messages[end] === "object" && + "role" in messages[end] && + messages[end].role === "tool" + ) { + end++; + } + messages.splice( + index, + end - index, + ...messages.slice(index, end).sort((left, right) => { + const leftName = + left !== null && + typeof left === "object" && + "name" in left && + typeof left.name === "string" + ? left.name + : ""; + const rightName = + right !== null && + typeof right === "object" && + "name" in right && + typeof right.name === "string" + ? right.name + : ""; + return leftName.localeCompare(rightName); + }), + ); + index = end - 1; + } + } + return { + ...request, + body: { + kind: "json", + value: { + ...body, + messages, + tools: body.tools.filter( + (tool) => + tool === null || + typeof tool !== "object" || + !("function" in tool) || + tool.function === null || + typeof tool.function !== "object" || + !("name" in tool.function) || + typeof tool.function.name !== "string" || + !EVE_BUILTIN_TOOLS.has(tool.function.name), + ), + }, + }, + }; + }, + }, +]; /** @type {import("@braintrust/seinfeld").RedactionSpec} */ export const redact = [ diff --git a/e2e/scenarios/eve-instrumentation/package.json b/e2e/scenarios/eve-instrumentation/package.json index 202841885..93b789343 100644 --- a/e2e/scenarios/eve-instrumentation/package.json +++ b/e2e/scenarios/eve-instrumentation/package.json @@ -14,9 +14,9 @@ }, "dependencies": { "@openrouter/ai-sdk-provider": "3.0.0", - "ai": "7.0.34", - "eve-v0": "npm:eve@0.22.1", - "eve-v0-latest": "npm:eve@0.27.6", + "ai": "7.0.68", + "eve-v0": "npm:eve@0.34.0", + "eve-v0-latest": "npm:eve@0.39.1", "zod": "4.3.6" } } diff --git a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml index c3cbc645d..e6b942a7a 100644 --- a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml @@ -10,36 +10,36 @@ importers: dependencies: '@openrouter/ai-sdk-provider': specifier: 3.0.0 - version: 3.0.0(ai@7.0.34(zod@4.3.6))(zod@4.3.6) + version: 3.0.0(ai@7.0.68(zod@4.3.6))(zod@4.3.6) ai: - specifier: 7.0.34 - version: 7.0.34(zod@4.3.6) + specifier: 7.0.68 + version: 7.0.68(zod@4.3.6) eve-v0: - specifier: npm:eve@0.22.1 - version: eve@0.22.1(ai@7.0.34(zod@4.3.6)) + specifier: npm:eve@0.34.0 + version: eve@0.34.0(ai@7.0.68(zod@4.3.6)) eve-v0-latest: - specifier: npm:eve@0.27.6 - version: eve@0.27.6(ai@7.0.34(zod@4.3.6)) + specifier: npm:eve@0.39.1 + version: eve@0.39.1(ai@7.0.68(zod@4.3.6)) zod: specifier: 4.3.6 version: 4.3.6 packages: - '@ai-sdk/gateway@4.0.26': - resolution: {integrity: sha512-b/nc3COKtk8IxzgcCi418IoZFky/Bw+Jg8+J0/SBBnu/mOXA4bkIKCu81coEtJAWuH2g5Fvvsa0ar7EpmzNWTw==} + '@ai-sdk/gateway@4.0.54': + resolution: {integrity: sha512-x4fAXDqCtYzB/M5vsIQLYcyrzpJuaRgcIwDSw+lpTMMbgH19fU3ds75GSlHNLzfx6Z5yL4Z9+EMr0GJcqVy9QA==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider-utils@5.0.12': - resolution: {integrity: sha512-bbhlOgHeYwrIGheLkM6fhS8hVger8uFPmcOLg+kxc9EFh7y30XYorWhthlYAgpadO3SJhFZrIcEknN7qEqEVvA==} + '@ai-sdk/provider-utils@5.0.27': + resolution: {integrity: sha512-EzAn4pdgG5g0xXtH6lE2zyNmfjDQIDjATkfqzuidEI35g++hh4+07vnjzkT/RmGmIClPZiRj/Q2GMPV2V7mkHw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 - '@ai-sdk/provider@4.0.3': - resolution: {integrity: sha512-e0CpNWJUY7OxAFAnCZkw+ri9QOHWwTs1tXP42782KFGCU07qt8NiXCrCVowyCB5dP2r5/Uls+g2oPd8kOJn9dw==} + '@ai-sdk/provider@4.0.7': + resolution: {integrity: sha512-6or44XprPzKbr8zkmzosowSE0pxkvJcoojBL+mCZvPUt3kvXp3XSNqeVun9golb1acEfSo6yaEBRT18h2VU+1Q==} engines: {node: '>=22'} '@emnapi/core@1.11.1': @@ -178,8 +178,8 @@ packages: '@workflow/serde@4.1.0': resolution: {integrity: sha512-pav4F2BoirECWR7Nf1TKt+2eETcBj7jj4cBefQ8VXQCA6NPkaKeLfj/zMgi+3zYV5ZIBT4GuUiphsj0/b9hPQQ==} - ai@7.0.34: - resolution: {integrity: sha512-jDqclWYqPGFKcUG4CQiKcBiwi5XqVMqvYy6eJdujVpvE1SDoB6j7RtjrGYz3Bn5B1dzTj1StAlrOebY//WK9/Q==} + ai@7.0.68: + resolution: {integrity: sha512-9QuZOT77wzoxxUC0NcueXhCo3HUHA/1pApIJ9VRyE+9/K+3Innkq4kVhrd9aEnwIviJz2Nga063m+UTsPSdOyw==} engines: {node: '>=22'} peerDependencies: zod: ^3.25.76 || ^4.1.8 @@ -237,13 +237,13 @@ packages: wrangler: optional: true - eve@0.22.1: - resolution: {integrity: sha512-4nqy8SEJpS94fM8sxd0Or+irvf/CmS0qENxouKyyegst3Qtn06xZbuSEoI1p7i4wB6sprr95udXh7Ulct6ABrA==} + eve@0.34.0: + resolution: {integrity: sha512-zgrYIbm8u78Furxfdyzaknf5AxFCzwmTGfLEvgax00BUNXu3eebaos8igjYQyM4q5JAxlWAoZCJuItE6r2S0cw==} engines: {node: '>=24'} hasBin: true peerDependencies: '@opentelemetry/api': ^1.0.0 - ai: ^7.0.0 + ai: ^7.0.58 braintrust: ^3.0.0 just-bash: ^3.0.0 microsandbox: ^0.5.0 @@ -257,13 +257,13 @@ packages: microsandbox: optional: true - eve@0.27.6: - resolution: {integrity: sha512-t2Xr/nlVGc0+Fy9fav8p/CJrwMSQM0zyuOGY1F1XapN0UFssGYBGa6Ml/Q5OnowvWwsGH4m0Tf2QAccayVCvtw==} + eve@0.39.1: + resolution: {integrity: sha512-HTd1A3/GsRKNpJ+6WEW4OEPAhWAId5IZ769hjuY3eJqj3cWeaEvBUDZSjUFjSDit63aqbfur0u7AH1m0JNr7EQ==} engines: {node: '>=24'} hasBin: true peerDependencies: '@opentelemetry/api': ^1.0.0 - ai: ^7.0.34 + ai: ^7.0.58 braintrust: ^3.0.0 just-bash: ^3.0.0 microsandbox: ^0.5.0 @@ -365,6 +365,14 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + undici@7.29.0: + resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} + engines: {node: '>=20.18.1'} + + undici@8.9.0: + resolution: {integrity: sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==} + engines: {node: '>=22.19.0'} + unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -447,22 +455,23 @@ packages: snapshots: - '@ai-sdk/gateway@4.0.26(zod@4.3.6)': + '@ai-sdk/gateway@4.0.54(zod@4.3.6)': dependencies: - '@ai-sdk/provider': 4.0.3 - '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.27(zod@4.3.6) '@vercel/oidc': 3.2.0 zod: 4.3.6 - '@ai-sdk/provider-utils@5.0.12(zod@4.3.6)': + '@ai-sdk/provider-utils@5.0.27(zod@4.3.6)': dependencies: - '@ai-sdk/provider': 4.0.3 + '@ai-sdk/provider': 4.0.7 '@standard-schema/spec': 1.1.0 '@workflow/serde': 4.1.0 eventsource-parser: 3.1.0 + undici: 7.29.0 zod: 4.3.6 - '@ai-sdk/provider@4.0.3': + '@ai-sdk/provider@4.0.7': dependencies: json-schema: 0.4.0 @@ -489,9 +498,9 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true - '@openrouter/ai-sdk-provider@3.0.0(ai@7.0.34(zod@4.3.6))(zod@4.3.6)': + '@openrouter/ai-sdk-provider@3.0.0(ai@7.0.68(zod@4.3.6))(zod@4.3.6)': dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) zod: 4.3.6 '@oxc-project/types@0.138.0': {} @@ -558,11 +567,11 @@ snapshots: '@workflow/serde@4.1.0': {} - ai@7.0.34(zod@4.3.6): + ai@7.0.68(zod@4.3.6): dependencies: - '@ai-sdk/gateway': 4.0.26(zod@4.3.6) - '@ai-sdk/provider': 4.0.3 - '@ai-sdk/provider-utils': 5.0.12(zod@4.3.6) + '@ai-sdk/gateway': 4.0.54(zod@4.3.6) + '@ai-sdk/provider': 4.0.7 + '@ai-sdk/provider-utils': 5.0.27(zod@4.3.6) zod: 4.3.6 consola@3.4.2: {} @@ -580,10 +589,11 @@ snapshots: httpxy: 0.5.4 srvx: 0.11.21 - eve@0.22.1(ai@7.0.34(zod@4.3.6)): + eve@0.34.0(ai@7.0.68(zod@4.3.6)): dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) nitro: 3.0.260610-beta + undici: 8.9.0 transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -624,10 +634,11 @@ snapshots: - xml2js - zephyr-agent - eve@0.27.6(ai@7.0.34(zod@4.3.6)): + eve@0.39.1(ai@7.0.68(zod@4.3.6)): dependencies: - ai: 7.0.34(zod@4.3.6) + ai: 7.0.68(zod@4.3.6) nitro: 3.0.260610-beta + undici: 8.9.0 transitivePeerDependencies: - '@azure/app-configuration' - '@azure/cosmos' @@ -773,6 +784,10 @@ snapshots: tslib@2.8.1: optional: true + undici@7.29.0: {} + + undici@8.9.0: {} + unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 diff --git a/e2e/scenarios/eve-instrumentation/scenario.test.ts b/e2e/scenarios/eve-instrumentation/scenario.test.ts index 4d65888c9..2fcd1ad48 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.test.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.test.ts @@ -26,7 +26,7 @@ const eveScenarios = await Promise.all( [ { dependencyName: "eve-v0", - label: "v0 pinned", + label: "v0.34 pinned", variantKey: "eve-v0", }, { @@ -137,12 +137,16 @@ describe.sequential("eve instrumentation variants", () => { "read", secondRoot?.span.id, ); - + const secondChildSearch = findLatestChildSpan( + events, + "search", + secondChildTurn?.span.id, + ); expect(findAllSpans(events, "eve.session")).toEqual([]); expect(turns).toHaveLength(4); expect( turns.filter((turn) => turn.span.parentIds.length === 0), - ).toHaveLength(2); + ).toEqual([root, secondRoot]); expect(new Set(turns.map((turn) => turn.span.rootId)).size).toBe(2); expect(root).toBeDefined(); @@ -155,10 +159,21 @@ describe.sequential("eve instrumentation variants", () => { }); expect(root?.metadata).not.toHaveProperty("model"); expect(root?.metadata).not.toHaveProperty("provider"); - expect(root?.metrics?.completion_tokens).toEqual(expect.any(Number)); - expect(root?.metrics?.prompt_tokens).toEqual(expect.any(Number)); - expect(root?.metrics?.tokens).toEqual(expect.any(Number)); expect(root?.output).toContain("Final answer from read"); + expect(root?.metrics).toMatchObject({ + completion_tokens: steps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: steps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: steps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(steps).toHaveLength(2); expect(steps.map((step) => step.span.type)).toEqual(["llm", "llm"]); @@ -191,11 +206,14 @@ describe.sequential("eve instrumentation variants", () => { expect(step.input[0]).toMatchObject({ role: "system" }); expect(step.metadata).toMatchObject({ "eve.session_id": root?.metadata?.["eve.session_id"], + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", scenario: "eve-instrumentation", testRunId: expect.any(String), }); - expect(step.metadata).not.toHaveProperty("model"); - expect(step.metadata).not.toHaveProperty("provider"); + expect(step.metrics?.completion_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.tokens).toEqual(expect.any(Number)); } expect(researcher).toBeDefined(); @@ -217,14 +235,28 @@ describe.sequential("eve instrumentation variants", () => { expect(childTurn?.span.rootId).toEqual(root?.span.rootId); expect(childTurn?.metadata).toMatchObject({ "eve.session_id": expect.any(String), - model: "qwen/qwen3-30b-a3b", - provider: "openrouter", scenario: "eve-instrumentation", testRunId: expect.any(String), }); + expect(childTurn?.metadata).not.toHaveProperty("model"); + expect(childTurn?.metadata).not.toHaveProperty("provider"); expect(childTurn?.metadata?.["eve.session_id"]).not.toEqual( root?.metadata?.["eve.session_id"], ); + expect(childTurn?.metrics).toMatchObject({ + completion_tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: childSteps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(childSteps).toHaveLength(2); for (const step of childSteps) { @@ -242,6 +274,9 @@ describe.sequential("eve instrumentation variants", () => { scenario: "eve-instrumentation", testRunId: expect.any(String), }); + expect(step.metrics?.completion_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.prompt_tokens).toEqual(expect.any(Number)); + expect(step.metrics?.tokens).toEqual(expect.any(Number)); expect(step.output).toMatchObject([ { message: { @@ -286,74 +321,81 @@ describe.sequential("eve instrumentation variants", () => { expect(secondRoot?.span.rootId).not.toEqual(root?.span.rootId); expect(secondRoot?.metadata).toMatchObject({ "eve.session_id": root?.metadata?.["eve.session_id"], + scenario: "eve-instrumentation", + testRunId: expect.any(String), }); - expect(secondRoot?.metadata).not.toHaveProperty("model"); - expect(secondRoot?.metadata).not.toHaveProperty("provider"); expect(secondRoot?.output).toContain("Final answer from read"); + expect(secondRoot?.metrics).toMatchObject({ + completion_tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.completion_tokens ?? 0), + 0, + ), + prompt_tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.prompt_tokens ?? 0), + 0, + ), + tokens: secondSteps.reduce( + (total, step) => total + (step.metrics?.tokens ?? 0), + 0, + ), + }); expect(secondSteps).toHaveLength(2); - expect(secondSteps.map((step) => step.span.type)).toEqual([ - "llm", - "llm", - ]); for (const step of secondSteps) { + expect(step.span.parentIds).toEqual([secondRoot?.span.id]); + expect(step.span.rootId).toEqual(secondRoot?.span.rootId); expect(step.metadata).toMatchObject({ "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - scenario: "eve-instrumentation", - testRunId: expect.any(String), + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", }); - expect(step.metadata).not.toHaveProperty("model"); - expect(step.metadata).not.toHaveProperty("provider"); } - expect(secondSteps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - reasoning: [{ content: expect.any(String) }], - tool_calls: [ - { function: { name: "researcher" }, type: "function" }, - { function: { name: "read" }, type: "function" }, - ], - }, - }, - ]); - expect(secondSteps[1]?.output).toMatchObject([ - { - finish_reason: "stop", - message: { - reasoning: [{ content: expect.any(String) }], - }, - }, - ]); - expect(secondResearcher?.span.type).toBe("tool"); - expect(secondResearcher?.span.ended).toBe(true); expect(secondResearcher?.span.parentIds).toEqual([secondRoot?.span.id]); - expect(secondResearcher?.metadata).toMatchObject({ - "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - }); expect(secondChildTurn?.span.parentIds).toEqual([ secondResearcher?.span.id, ]); expect(secondChildTurn?.span.rootId).toEqual(secondRoot?.span.rootId); - expect(secondChildTurn?.metadata).toMatchObject({ - "eve.session_id": expect.any(String), - model: "qwen/qwen3-30b-a3b", - provider: "openrouter", - }); - expect(secondChildTurn?.metadata?.["eve.session_id"]).not.toEqual( - secondRoot?.metadata?.["eve.session_id"], - ); - expect(secondRead?.span.type).toBe("tool"); - expect(secondRead?.span.ended).toBe(true); + expect(secondChildTurn?.span.rootId).not.toEqual(root?.span.rootId); expect(secondRead?.span.parentIds).toEqual([secondRoot?.span.id]); - expect(secondRead?.metadata).toMatchObject({ - "eve.session_id": secondRoot?.metadata?.["eve.session_id"], - }); for (const event of events) { expect(spanInstrumentationName(event)).toBe("eve"); } const rawRows = payloads.flatMap((payload) => payload.rows); + for (const span of [ + ...findAllSpans(events, "eve.step"), + researcher, + childSearch, + read, + secondResearcher, + secondRead, + secondChildSearch, + ]) { + expect(span).toBeDefined(); + expect( + rawRows.filter( + (row) => + row.id === span?.row.id && + Object.prototype.hasOwnProperty.call(row, "input"), + ), + ).toHaveLength(1); + expect( + rawRows.filter( + (row) => + row.id === span?.row.id && + Object.prototype.hasOwnProperty.call(row, "metadata"), + ), + ).toHaveLength(1); + } + for (const turn of turns) { + expect( + rawRows.filter( + (row) => + row.id === turn.row.id && + Object.prototype.hasOwnProperty.call(row, "metadata"), + ), + ).toHaveLength(1); + } for (const step of findAllSpans(events, "eve.step")) { expect( rawRows.filter( diff --git a/e2e/scenarios/eve-instrumentation/scenario.ts b/e2e/scenarios/eve-instrumentation/scenario.ts index 48efc7bea..ca1c92027 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.ts @@ -65,13 +65,10 @@ async function main() { } const body = (await response.json()) as { - continuationToken?: string; sessionId?: string; }; - if (!body.sessionId || !body.continuationToken) { - throw new Error( - `Eve session create did not return a sessionId and continuationToken`, - ); + if (!body.sessionId) { + throw new Error(`Eve session create did not return a sessionId`); } const seenSessionIds = new Set([body.sessionId]); @@ -81,11 +78,13 @@ async function main() { seenSessionIds, "session.waiting", ); + // Eve emits session.waiting just before its durable session snapshot is + // visible to the continuation route. + await new Promise((resolve) => setTimeout(resolve, 1000)); const followUp = await fetch( `${baseUrl}/eve/v1/session/${body.sessionId}`, { body: JSON.stringify({ - continuationToken: body.continuationToken, message: "Run the Braintrust Eve instrumentation e2e scenario again", }), headers: { "content-type": "application/json" }, @@ -213,6 +212,7 @@ async function streamUntil( const decoder = new TextDecoder(); let buffer = ""; let nextIndex = startIndex; + let turnCompleted = false; try { while (true) { const { done, value } = await reader.read(); @@ -228,7 +228,10 @@ async function streamUntil( continue; } const event = JSON.parse(trimmed) as { - data?: { childSessionId?: string; message?: string }; + data?: { + childSessionId?: string; + message?: string; + }; type?: string; }; nextIndex++; @@ -256,7 +259,13 @@ async function streamUntil( ).then(() => undefined), ); } - if (event.type === until) { + if (event.type === "turn.completed") { + turnCompleted = true; + } + if ( + event.type === until && + (until !== "session.waiting" || turnCompleted) + ) { await Promise.all(childStreams); return nextIndex; } diff --git a/e2e/scenarios/flue-instrumentation/.flue/app.mjs b/e2e/scenarios/flue-instrumentation/.flue/app.mjs index 900402277..5d3ce1360 100644 --- a/e2e/scenarios/flue-instrumentation/.flue/app.mjs +++ b/e2e/scenarios/flue-instrumentation/.flue/app.mjs @@ -1,12 +1,12 @@ -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname } from "node:path"; -import { flush, initLogger } from "braintrust"; +import { braintrustFlueInstrumentation, flush, initLogger } from "braintrust"; +import { Hono } from "hono"; const runtimePackageName = process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "@flue/runtime"; -const { configureProvider, flue, observe } = await import( - `${runtimePackageName}/app` -); +const [{ instrument, registerProvider }, { flue }] = await Promise.all([ + import(runtimePackageName), + import(`${runtimePackageName}/routing`), +]); function projectName() { const configured = process.env.BRAINTRUST_E2E_PROJECT_NAME; @@ -19,84 +19,24 @@ function projectName() { initLogger({ projectName: projectName() }); -const exitProcess = process.exit.bind(process); -if (process.env.FLUE_E2E_FLUSH_FILE) { - let isExiting = false; - process.exit = (code) => { - if (isExiting) { - return exitProcess(code); - } - isExiting = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - exitProcess(code); - }); - }; -} - -if (process.env.FLUE_E2E_EXPLICIT_OBSERVE === "1") { - const { braintrustFlueObserver } = await import("braintrust"); - observe(braintrustFlueObserver); -} +instrument(braintrustFlueInstrumentation()); const openAIBaseUrl = process.env.OPENAI_BASE_URL ?? process.env.BRAINTRUST_E2E_MODEL_BASE_URL; if (openAIBaseUrl) { - configureProvider("openai", { baseUrl: openAIBaseUrl }); + registerProvider("openai", { baseUrl: openAIBaseUrl }); } const anthropicBaseUrl = process.env.ANTHROPIC_BASE_URL; if (anthropicBaseUrl) { - configureProvider("anthropic", { + registerProvider("anthropic", { apiKey: process.env.ANTHROPIC_API_KEY ?? "test-key", baseUrl: anthropicBaseUrl, }); } -let didScheduleFlush = false; -function scheduleFinalFlush(exitAfterFlush = false) { - if (didScheduleFlush) { - return; - } - didScheduleFlush = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - if (exitAfterFlush) { - exitProcess(0); - } - }); -} - -process.on("SIGTERM", () => { - scheduleFinalFlush(true); -}); - -process.on("beforeExit", () => { - scheduleFinalFlush(); -}); - -const app = flue(); - -async function flushBeforeExit() { - await new Promise((resolve) => setTimeout(resolve, 250)); - await flush(); - if (process.env.FLUE_E2E_FLUSH_FILE) { - await mkdir(dirname(process.env.FLUE_E2E_FLUSH_FILE), { - recursive: true, - }); - await writeFile(process.env.FLUE_E2E_FLUSH_FILE, "ok"); - } -} +const app = new Hono(); +app.route("/", flue()); export default { async fetch(request, env, ctx) { diff --git a/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs b/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs index dd15a5516..509be2b53 100644 --- a/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs +++ b/e2e/scenarios/flue-instrumentation/.flue/workflows/instrumentation.mjs @@ -1,4 +1,5 @@ import { traced } from "braintrust"; +import * as v from "valibot"; import { FLUE_MODEL, FLUE_REASONING_MODEL, @@ -7,10 +8,11 @@ import { const runtimePackageName = process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "@flue/runtime"; -const [{ createAgent, Type }, { local }] = await Promise.all([ - import(runtimePackageName), - import(`${runtimePackageName}/node`), -]); +const [{ defineAgent, defineTool, defineWorkflow }, { local }] = + await Promise.all([ + import(runtimePackageName), + import(`${runtimePackageName}/node`), + ]); function flueModel() { return process.env.FLUE_E2E_MODEL ?? FLUE_MODEL; @@ -34,7 +36,7 @@ function flueReasoningThinkingLevel() { return process.env.FLUE_E2E_REASONING_THINKING_LEVEL ?? "medium"; } -const flueE2EAgent = createAgent(() => ({ +const flueE2EAgent = defineAgent(() => ({ compaction: { keepRecentTokens: 1, reserveTokens: 64, @@ -51,10 +53,14 @@ const flueE2EAgent = createAgent(() => ({ thinkingLevel: "off", })); -const lookupTool = { +const lookupTool = defineTool({ description: "Return a deterministic lookup result with an id needed by web_search.", - execute: async (args) => { + input: v.object({ + query: v.string(), + }), + name: "lookup", + run: async ({ input }) => { await traced( async (span) => { span.log({ output: "lookup-active" }); @@ -69,109 +75,109 @@ const lookupTool = { }, ); - return JSON.stringify({ + return { id: "flue-session-2026", - query: args.query, + query: input.query, topic: "session instrumentation", - }); + }; }, - name: "lookup", - parameters: Type.Object({ - query: Type.String(), - }), -}; +}); -const webSearchTool = { +const webSearchTool = defineTool({ description: "Search a deterministic local web index. Requires the id returned by lookup.", - execute: async (args) => - JSON.stringify({ - lookupId: args.lookupId, - query: args.query, - results: [ - { - title: "Flue reasoning stream instrumentation", - url: "https://example.test/flue/reasoning-streams", - }, - ], - }), + input: v.object({ + lookupId: v.string(), + query: v.string(), + }), name: "web_search", - parameters: Type.Object({ - lookupId: Type.String(), - query: Type.String(), + run: async ({ input }) => ({ + lookupId: input.lookupId, + query: input.query, + results: [ + { + title: "Flue reasoning stream instrumentation", + url: "https://example.test/flue/reasoning-streams", + }, + ], }), -}; +}); -const summarizeSourceTool = { +const summarizeSourceTool = defineTool({ description: "Summarize the selected deterministic source after web_search returns a URL.", - execute: async (args) => - JSON.stringify({ - summary: - "Flue emits reasoning, tool execution, and LLM turn events separately.", - url: args.url, - }), + input: v.object({ + url: v.string(), + }), name: "summarize_source", - parameters: Type.Object({ - url: Type.String(), + run: async ({ input }) => ({ + summary: + "Flue emits reasoning, tool execution, and LLM turn events separately.", + url: input.url, }), -}; +}); export async function route(_ctx, next) { await next(); } -export async function run({ init, payload }) { - await traced( - async (span) => { - span.log({ output: "active" }); - }, - { - name: "flue.workflowCurrentProbe", - event: { - metadata: { - scenario: SCENARIO_NAME, +export default defineWorkflow({ + agent: flueE2EAgent, + input: v.object({ + scenario: v.optional(v.string()), + metadata: v.optional(v.record(v.string(), v.unknown())), + }), + async run({ harness, input }) { + await traced( + async (span) => { + span.log({ output: "active" }); + }, + { + name: "flue.workflowCurrentProbe", + event: { + metadata: { + scenario: SCENARIO_NAME, + }, }, }, - }, - ); + ); - const harness = await init(flueE2EAgent, { name: "default" }); - const session = await harness.session("main"); - const skillSession = await harness.session("skill"); - const taskSession = await harness.session("task"); - - await session.prompt( - [ - "Complete this instrumented research flow.", - "Call exactly one tool per turn and wait for each tool result before choosing the next tool.", - 'Step 1: call lookup with query "flue instrumentation".', - 'Step 2: use the lookup result id as lookupId and call web_search with query "Braintrust Flue reasoning stream instrumentation".', - "Step 3: use the first web_search result url and call summarize_source.", - "After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - ].join(" "), - { - model: fluePromptModel(), - thinkingLevel: fluePromptThinkingLevel(), - tools: [lookupTool, webSearchTool, summarizeSourceTool], - }, - ); + const session = await harness.session("main"); + const skillSession = await harness.session("skill"); + const taskSession = await harness.session("task"); + + await session.prompt( + [ + "Complete this instrumented research flow.", + "Call exactly one tool per turn and wait for each tool result before choosing the next tool.", + 'Step 1: call lookup with query "flue instrumentation".', + 'Step 2: use the lookup result id as lookupId and call web_search with query "Braintrust Flue reasoning stream instrumentation".', + "Step 3: use the first web_search result url and call summarize_source.", + "After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", + ].join(" "), + { + model: fluePromptModel(), + thinkingLevel: fluePromptThinkingLevel(), + tools: [lookupTool, webSearchTool, summarizeSourceTool], + }, + ); - await skillSession.skill("e2e-flue-skill", { - args: { marker: "SKILL_DONE" }, - model: flueReasoningModel(), - thinkingLevel: "off", - }); + await skillSession.skill("e2e-flue-skill", { + args: { marker: "SKILL_DONE" }, + model: flueReasoningModel(), + thinkingLevel: "off", + }); - await taskSession.task("Reply with exactly TASK_DONE and no other text.", { - model: FLUE_MODEL, - thinkingLevel: "off", - }); + await taskSession.task("Reply with exactly TASK_DONE and no other text.", { + model: FLUE_MODEL, + thinkingLevel: "off", + }); - await session.compact(); + await session.compact(); - return { - scenario: payload?.scenario ?? SCENARIO_NAME, - status: "done", - }; -} + return { + scenario: input.scenario ?? SCENARIO_NAME, + status: "done", + }; + }, +}); diff --git a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json b/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json deleted file mode 100644 index 996df5098..000000000 --- a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-0.cassette.json +++ /dev/null @@ -1,3727 +0,0 @@ -{ - "entries": [ - { - "callIndex": 0, - "id": "fcd9d51c145fb90b", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:40:49.578Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01A7K7rFFvpGEkYsKHme6rCx\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":10,\"cache_creation_input_tokens\":2222,\"cache_read_input_tokens\":0,\"cache_creation\":{\"ephemeral_5m_input_tokens\":2222,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":8,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"The user wants me to complete a research\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" flow with exactly three steps:\\n1. Call lookup with query \\\"flue instrumentation\\\"\\n2. Call web_search with the\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" lookupId from step 1 and query \\\"Braintrust Flue reasoning stream instrumentation\\\"\\n3. Call summarize_source with the first URL from step 2\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"\\n4. Reply with exactly \\\"PROMPT_DONE\\\"\\n\\nThe user emphasizes I should call exactly one tool per turn and wait for each result\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\". Let me start with step 1.\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"I'll complete this instrumented research flow step\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\" by step. Starting with Step 1:\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01MbotvqKgQV4ecxQ566vyuC\",\"name\":\"lookup\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"query\\\": \\\"flue instrumentation\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":10,\"cache_creation_input_tokens\":2222,\"cache_read_input_tokens\":0,\"output_tokens\":196,\"output_tokens_details\":{\"thinking_tokens\":120}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:45Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:45Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:45Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:45Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac490ea0c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:40:47 GMT", - "request-id": "req_011CbV4Yrzdciu7aM7XyPJvu", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-def621f99a586b9f40e35c15380d1f90-62267612b2e498ea-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1355", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "ff22a182ea5a9c06", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:40:53.810Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "input": { - "query": "flue instrumentation" - }, - "name": "lookup", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "is_error": false, - "tool_use_id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HZbhxuwLjyFGr1czSAWHTh\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":236,\"cache_read_input_tokens\":2222,\"cache_creation\":{\"ephemeral_5m_input_tokens\":236,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Good\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\", I got the lookup result with id \\\"flue-session-2026\\\". Now I need to proceed to step 2:\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" call web_search with this lookupId and the query \\\"Braintrust Flue reasoning stream instrumentation\\\".\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Now proceeding to Step 2:\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_018rc1ig7vXeron3rLfpUtRT\",\"name\":\"web_search\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"lookupId\\\": \\\"flue-session-2026\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\", \\\"query\\\": \\\"Braintrust Flue reasoning stream instru\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"mentation\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":236,\"cache_read_input_tokens\":2222,\"output_tokens\":158,\"output_tokens_details\":{\"thinking_tokens\":59}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:49Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:49Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:49Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:49Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac631dbac301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:40:52 GMT", - "request-id": "req_011CbV4ZAqRr9QtjYnVyY69n", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-68d6456f3077bdeb0822f73c39a51d2d-5be926a484ba813d-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2225", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 2, - "id": "f1d50a1e6c760d6c", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:40:58.541Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "input": { - "query": "flue instrumentation" - }, - "name": "lookup", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "is_error": false, - "tool_use_id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "name": "web_search", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "is_error": false, - "tool_use_id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HJoo9ncrvNPVcaq2UWN45L\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":234,\"cache_read_input_tokens\":2458,\"cache_creation\":{\"ephemeral_5m_input_tokens\":234,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":4,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Perfect! I got\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" the web_search results with one result. The first (and only) result has the URL\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\" \\\"https://example.test/flue/reasoning-streams\\\". Now I need to proceed to step 3: call summarize_source with this URL.\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"Now proceeding to Step 3:\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":2,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01MRjt3UmRqrUYHt1ZC8gND9\",\"name\":\"summarize_source\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"url\\\": \\\"https://example.test/flue/reasoning-streams\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":2,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":2}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":234,\"cache_read_input_tokens\":2458,\"output_tokens\":138,\"output_tokens_details\":{\"thinking_tokens\":61}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2998000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:54Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:54Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:54Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3598000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:54Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac7d4a1fc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:40:56 GMT", - "request-id": "req_011CbV4ZUiiFpwboN3YtdzkH", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-6392939f911bfe9568c361c78254c370-cb7c6141cf28bb77-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2757", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 3, - "id": "305957222aa1f6e9", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:01.029Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "input": { - "query": "flue instrumentation" - }, - "name": "lookup", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "is_error": false, - "tool_use_id": "toolu_01MbotvqKgQV4ecxQ566vyuC", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "name": "web_search", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "is_error": false, - "tool_use_id": "toolu_018rc1ig7vXeron3rLfpUtRT", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "signature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "id": "toolu_01MRjt3UmRqrUYHt1ZC8gND9", - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "name": "summarize_source", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "is_error": false, - "tool_use_id": "toolu_01MRjt3UmRqrUYHt1ZC8gND9", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "budget_tokens": 8192, - "display": "summarized", - "type": "enabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "name": "lookup" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "name": "web_search" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "name": "summarize_source" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_019YC1YEtmEwfw9dyuY3MPdx\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":194,\"cache_read_input_tokens\":2692,\"cache_creation\":{\"ephemeral_5m_input_tokens\":194,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":1,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"thinking\",\"thinking\":\"\",\"signature\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"Perfect\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \\\"PROMPT_DONE\\\" and no other text.\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"thinking_delta\",\"thinking\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"signature_delta\",\"signature\":\"Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"text_delta\",\"text\":\"PROMPT_DONE\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":13,\"cache_creation_input_tokens\":194,\"cache_read_input_tokens\":2692,\"output_tokens\":51,\"output_tokens_details\":{\"thinking_tokens\":40}}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2998000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:40:58Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:40:58Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:40:58Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3598000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:40:58Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dac9afd8fc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:00 GMT", - "request-id": "req_011CbV4Zq3L7gJxP1Y46qapx", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-e0b4a5e0f6b8f2625262dec81a52e61e-c551de5bb2f6f203-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1421", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 4, - "id": "3d93fe37f72a947f", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:05.303Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01SoNm9ntxkXQkRrBSidrXSB\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":3,\"cache_creation_input_tokens\":670,\"cache_read_input_tokens\":1099,\"cache_creation\":{\"ephemeral_5m_input_tokens\":670,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":5,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"I'll run the e\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"2e-flue-skill. Let me first read the skill file to understand its instructions.\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":1,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_0116y5v4MmMkEaDkjkEPPjTH\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/e2e-flue\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"-skill.md\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":1,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":1}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":3,\"cache_creation_input_tokens\":670,\"cache_read_input_tokens\":1099,\"output_tokens\":89}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:01Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:01Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:01Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:01Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacab0e1fc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:04 GMT", - "request-id": "req_011CbV4a2cMY5XByBTCzUoHh", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-e05c9702b7456a4b1c655ed13d92ba42-b48b861130669b0c-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2864", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 5, - "id": "fb4ed2bb1569d980", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:08.068Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HWMEiz1LLcSzQG6KkyGVxZ\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":7,\"cache_creation_input_tokens\":183,\"cache_read_input_tokens\":1769,\"cache_creation\":{\"ephemeral_5m_input_tokens\":183,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":52,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01VLK49ybvQa2MZoZ8FxbJCx\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":7,\"cache_creation_input_tokens\":183,\"cache_read_input_tokens\":1769,\"output_tokens\":52}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:05Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:05Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:05Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:05Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacc65c8dc301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:07 GMT", - "request-id": "req_011CbV4aLhY55XXCvF766Z84", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-1467e6f052128317c79e4dcdcc34c0e2-95e633d304e0c3ad-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2046", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 6, - "id": "2d5b5e2519b5661b", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:10.832Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01RRobMzKrtpPcHZAGQrRZCo\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":66,\"cache_read_input_tokens\":1952,\"cache_creation\":{\"ephemeral_5m_input_tokens\":66,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":54,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_018pSd7nKSm3kfi2DVmzQPsA\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/skills\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":66,\"cache_read_input_tokens\":1952,\"output_tokens\":54}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:08Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:08Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:08Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:08Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacd76bc8c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:10 GMT", - "request-id": "req_011CbV4aYPzoGb5kYRUcefKG", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-eb3b7c53373bb1de39b3c714cb8004f5-e91254d86c0e7b64-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2024", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 7, - "id": "2cbc017f3c17c05f", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:13.081Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "input": { - "path": ".agents/skills" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "e2e-flue-skill", - "is_error": false, - "tool_use_id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01HaDyq5MaZpVHgVGhHJLmvs\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":74,\"cache_read_input_tokens\":2018,\"cache_creation\":{\"ephemeral_5m_input_tokens\":74,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":63,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_012TAbdNbuuFaDw1naUg1vbT\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/skills/e2e-flue-skill\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":74,\"cache_read_input_tokens\":2018,\"output_tokens\":63}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:11Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:11Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:11Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:11Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dace85ad2c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:13 GMT", - "request-id": "req_011CbV4ajy1g4mH27mS5Ju8N", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-93ed38ba66b829c67909054d6945c9cb-af18094065e38879-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1778", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 8, - "id": "4b885efa84390a1f", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:15.608Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "input": { - "path": ".agents/skills" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "e2e-flue-skill", - "is_error": false, - "tool_use_id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "SKILL.md", - "is_error": false, - "tool_use_id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01FCY15E5PvgZ9MKVwMLQfmY\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":79,\"cache_read_input_tokens\":2092,\"cache_creation\":{\"ephemeral_5m_input_tokens\":79,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":66,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"toolu_01WQVh4vmLxZQhqpoXRvUi68\",\"name\":\"read\",\"input\":{},\"caller\":{\"type\":\"direct\"}}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"path\\\": \\\".agents/skills/e2e-flue-skill/SKILL.md\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"\\\"}\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"tool_use\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":79,\"cache_read_input_tokens\":2092,\"output_tokens\":68}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:13Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:13Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:13Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:13Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dacf599b5c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:15 GMT", - "request-id": "req_011CbV4atzVFuefiAGaQnZig", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-75d6c58deb0a99da3a3ea03db1ae3fd9-01d8b31ff753a269-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "2118", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 9, - "id": "05dcf3cc09c5aa27", - "matchKey": "POST api.anthropic.com/v1/messages", - "recordedAt": "2026-05-28T13:41:17.354Z", - "request": { - "body": { - "kind": "json", - "value": { - "max_tokens": 64000, - "messages": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "ENOENT: no such file or directory, open '/Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "is_error": true, - "tool_use_id": "toolu_0116y5v4MmMkEaDkjkEPPjTH", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "input": { - "path": ".agents" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "skills", - "is_error": false, - "tool_use_id": "toolu_01VLK49ybvQa2MZoZ8FxbJCx", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "input": { - "path": ".agents/skills" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "e2e-flue-skill", - "is_error": false, - "tool_use_id": "toolu_018pSd7nKSm3kfi2DVmzQPsA", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "content": "SKILL.md", - "is_error": false, - "tool_use_id": "toolu_012TAbdNbuuFaDw1naUg1vbT", - "type": "tool_result" - } - ], - "role": "user" - }, - { - "content": [ - { - "id": "toolu_01WQVh4vmLxZQhqpoXRvUi68", - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "name": "read", - "type": "tool_use" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "cache_control": { - "type": "ephemeral" - }, - "content": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "is_error": false, - "tool_use_id": "toolu_01WQVh4vmLxZQhqpoXRvUi68", - "type": "tool_result" - } - ], - "role": "user" - } - ], - "model": "claude-sonnet-4-5-20250929", - "stream": true, - "system": [ - { - "cache_control": { - "type": "ephemeral" - }, - "text": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "type": "text" - } - ], - "thinking": { - "type": "disabled" - }, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "name": "read" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "name": "write" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "name": "edit" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "name": "bash" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "grep" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "name": "glob" - }, - { - "cache_control": { - "type": "ephemeral" - }, - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "eager_input_streaming": true, - "input_schema": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "name": "task" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.anthropic.com/v1/messages" - }, - "response": { - "body": { - "chunks": [ - "event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"model\":\"claude-sonnet-4-5-20250929\",\"id\":\"msg_01F7J8rewVTLnDMtxDcVdkiQ\",\"type\":\"message\",\"role\":\"assistant\",\"content\":[],\"stop_reason\":null,\"stop_sequence\":null,\"stop_details\":null,\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":135,\"cache_read_input_tokens\":2171,\"cache_creation\":{\"ephemeral_5m_input_tokens\":135,\"ephemeral_1h_input_tokens\":0},\"output_tokens\":3,\"service_tier\":\"standard\",\"inference_geo\":\"not_available\"}}}", - "event: content_block_start\ndata: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}", - "event: ping\ndata: {\"type\":\"ping\"}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"SKILL_\"}}", - "event: content_block_delta\ndata: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"DONE\"}}", - "event: content_block_stop\ndata: {\"type\":\"content_block_stop\",\"index\":0}", - "event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\",\"stop_sequence\":null,\"stop_details\":null},\"usage\":{\"input_tokens\":6,\"cache_creation_input_tokens\":135,\"cache_read_input_tokens\":2171,\"output_tokens\":8}}", - "event: message_stop\ndata: {\"type\":\"message_stop\"}" - ], - "kind": "sse" - }, - "headers": { - "anthropic-organization-id": "27796668-7351-40ac-acc4-024aee8995a5", - "anthropic-ratelimit-input-tokens-limit": "3000000", - "anthropic-ratelimit-input-tokens-remaining": "2999000", - "anthropic-ratelimit-input-tokens-reset": "2026-05-28T13:41:15Z", - "anthropic-ratelimit-output-tokens-limit": "600000", - "anthropic-ratelimit-output-tokens-remaining": "600000", - "anthropic-ratelimit-output-tokens-reset": "2026-05-28T13:41:15Z", - "anthropic-ratelimit-requests-limit": "20000", - "anthropic-ratelimit-requests-remaining": "19999", - "anthropic-ratelimit-requests-reset": "2026-05-28T13:41:15Z", - "anthropic-ratelimit-tokens-limit": "3600000", - "anthropic-ratelimit-tokens-remaining": "3599000", - "anthropic-ratelimit-tokens-reset": "2026-05-28T13:41:15Z", - "cache-control": "no-cache", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dad05abc5c301-VIE", - "connection": "keep-alive", - "content-encoding": "gzip", - "content-security-policy": "default-src 'none'; frame-ancestors 'none'", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:17 GMT", - "request-id": "req_011CbV4b62WtkEAN1uuw6ZTa", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "traceresponse": "00-a95abb0aa0dc3ddebd7bd3e84f6431b6-69727f41ec2a9c74-01", - "transfer-encoding": "chunked", - "vary": "Accept-Encoding", - "x-envoy-upstream-service-time": "1258", - "x-robots-tag": "none" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 0, - "id": "7c0e162d6794cf5d", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-28T13:41:20.282Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Thu, May 28, 2026\nWorking directory: /Users/lucaforstner/conductor/workspaces/braintrust-sdk-javascript/shanghai-v1/e2e/.bt-tmp/run-u9uXnv/scenarios/flue-instrumentation\n\nDirectory structure:\n.agents\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts", - "role": "system" - }, - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-4o-mini", - "prompt_cache_key": "[REDACTED]", - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_09b32b2821e9a4d8016a1845ff2ea88192a734cf6ba6abcb9d\",\"object\":\"response\",\"created_at\":1779975679,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"flue-e2e-instance::default::task:task:4cfd6e11-bac1-4736-9b87-b0\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_09b32b2821e9a4d8016a1845ff2ea88192a734cf6ba6abcb9d\",\"object\":\"response\",\"created_at\":1779975679,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"flue-e2e-instance::default::task:task:4cfd6e11-bac1-4736-9b87-b0\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"TASK\",\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"logprobs\":[],\"obfuscation\":\"pyb2YZ3HkAzp\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"logprobs\":[],\"obfuscation\":\"zDbKM0sYFZQ\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":6,\"text\":\"TASK_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"},\"sequence_number\":7}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":8}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_09b32b2821e9a4d8016a1845ff2ea88192a734cf6ba6abcb9d\",\"object\":\"response\",\"created_at\":1779975679,\"status\":\"completed\",\"background\":false,\"completed_at\":1779975680,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"flue-e2e-instance::default::task:task:4cfd6e11-bac1-4736-9b87-b0\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":804,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":808},\"user\":null,\"metadata\":{}},\"sequence_number\":9}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dad1668335adb-VIE", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:19 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "321", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_fdef3765899a4c91bc975a65974dd9bb" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "f12aff3913439248", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-05-28T13:41:21.738Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are a context summarization assistant. Your task is to read a conversation between a user and an AI coding assistant, then produce a structured summary following the exact format specified.\n\nDo NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.", - "role": "system" - }, - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "max_output_tokens": 32, - "model": "gpt-4o-mini", - "store": false, - "stream": true - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_01ecf68325f9e202016a184600a16881a08ded2a1961eda9f9\",\"object\":\"response\",\"created_at\":1779975680,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_01ecf68325f9e202016a184600a16881a08ded2a1961eda9f9\",\"object\":\"response\",\"created_at\":1779975680,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"##\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"d6v2dmzPZ7YChr\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Original\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"AQ3m9tJ\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Request\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"UKhv1LKl\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"5muwgiakPkjQduY\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"gYHJ7xqWx893Q\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" user\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"ta2Vk95LgSS\",\"output_index\":0,\"sequence_number\":9}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" requested\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"T18OOl\",\"output_index\":0,\"sequence_number\":10}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" to\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"CgWTkMgs6kW4s\",\"output_index\":0,\"sequence_number\":11}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" complete\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"PQ7u3KE\",\"output_index\":0,\"sequence_number\":12}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" an\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"fyFt8NlmxkL8T\",\"output_index\":0,\"sequence_number\":13}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" instrument\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"idBTA\",\"output_index\":0,\"sequence_number\":14}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ed\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"Jqp8Abx3NlEpYO\",\"output_index\":0,\"sequence_number\":15}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" research\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"2Z7W3UL\",\"output_index\":0,\"sequence_number\":16}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" flow\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"CWcmUwdOK0I\",\"output_index\":0,\"sequence_number\":17}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" consisting\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"5A8K0\",\"output_index\":0,\"sequence_number\":18}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" of\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"8mUUf5KvarpcJ\",\"output_index\":0,\"sequence_number\":19}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" three\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"RpVzByEpb2\",\"output_index\":0,\"sequence_number\":20}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" specific\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"9qbTunN\",\"output_index\":0,\"sequence_number\":21}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" steps\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"sRljhC5wZy\",\"output_index\":0,\"sequence_number\":22}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"LQiEbRM4fizUNQm\",\"output_index\":0,\"sequence_number\":23}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" involving\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"CFyfLz\",\"output_index\":0,\"sequence_number\":24}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" calling\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"E6cI70HR\",\"output_index\":0,\"sequence_number\":25}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" various\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"GFRuEgJV\",\"output_index\":0,\"sequence_number\":26}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" tools\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"X4Ttj2uHGX\",\"output_index\":0,\"sequence_number\":27}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" in\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"9vlDZO40VZwFr\",\"output_index\":0,\"sequence_number\":28}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" a\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"tWdPAInHFKDip7\",\"output_index\":0,\"sequence_number\":29}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" sequential\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"Rct1b\",\"output_index\":0,\"sequence_number\":30}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" manner\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"0x4W3URIt\",\"output_index\":0,\"sequence_number\":31}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\\n\\n\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"EscdvHRY9LVkB\",\"output_index\":0,\"sequence_number\":32}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"##\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"7fE3Nivp8RY0eK\",\"output_index\":0,\"sequence_number\":33}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Early\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"qsGDoylwsO\",\"output_index\":0,\"sequence_number\":34}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Progress\",\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"obfuscation\":\"WxaVUkk\",\"output_index\":0,\"sequence_number\":35}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":36,\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"},\"sequence_number\":37}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":38}", - "event: response.incomplete\ndata: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_01ecf68325f9e202016a184600a16881a08ded2a1961eda9f9\",\"object\":\"response\",\"created_at\":1779975680,\"status\":\"incomplete\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\\n\\n## Early Progress\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":694,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":32,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":726},\"user\":null,\"metadata\":{}},\"sequence_number\":39}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a02dad234c015adb-VIE", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Thu, 28 May 2026 13:41:20 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "90", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_2bb8f213d8be45a8a8c6368e34e80335" - }, - "status": 200, - "statusText": "OK" - } - } - ], - "meta": { - "createdAt": "2026-05-28T13:01:46.277Z" - } -} diff --git a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json b/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json deleted file mode 100644 index 69d4e8bd6..000000000 --- a/e2e/scenarios/flue-instrumentation/__cassettes__/flue-v0-8-latest.cassette.json +++ /dev/null @@ -1,5859 +0,0 @@ -{ - "entries": [ - { - "callIndex": 0, - "id": "7c0e162d6794cf5d", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:00.374Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b5f8ed48191ac5f4bcf9d7df429\",\"object\":\"response\",\"created_at\":1785748319,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b5f8ed48191ac5f4bcf9d7df429\",\"object\":\"response\",\"created_at\":1785748319,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_rBgonKxWHwhN9E8aQ4VobIOe\",\"name\":\"lookup\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"FJ9TowjZDi2V7k\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"query\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"s6VEwiE9UsM\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"6aWQMwZgQPiRf\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"fl\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"KUoAPUAlQ7cmX9\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"lQUgpuDVqWyLbo\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" instrumentation\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"obfuscation\":\"jkQ7s9o73nxqwB\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"query\\\":\\\"flue instrumentation\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"output_index\":0,\"sequence_number\":10}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"query\\\":\\\"flue instrumentation\\\"}\",\"call_id\":\"call_rBgonKxWHwhN9E8aQ4VobIOe\",\"name\":\"lookup\"},\"output_index\":0,\"sequence_number\":11}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b5f8ed48191ac5f4bcf9d7df429\",\"object\":\"response\",\"created_at\":1785748319,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748320,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_045458052dc10379016a705b6008408191b27653755cf5bd03\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"query\\\":\\\"flue instrumentation\\\"}\",\"call_id\":\"call_rBgonKxWHwhN9E8aQ4VobIOe\",\"name\":\"lookup\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1005,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":19,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1024},\"user\":null,\"metadata\":{}},\"sequence_number\":12}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432b38d617bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:11:59 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "572", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998539", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_3ad47b9556ec41a8bf046bb73633b660" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 1, - "id": "f12aff3913439248", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:01.038Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"query\":\"flue instrumentation\"}", - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "id": "fc_045458052dc10379016a705b6008408191b27653755cf5bd03", - "name": "lookup", - "type": "function_call" - }, - { - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "output": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6070c48191b73ad8a8f9b550f7\",\"object\":\"response\",\"created_at\":1785748320,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6070c48191b73ad8a8f9b550f7\",\"object\":\"response\",\"created_at\":1785748320,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_fGesVCba2aABMoxSNfVnDVnR\",\"name\":\"web_search\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"CXnKLapUymev31\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"lookup\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"hPsgf6WVxe\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Id\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"jPh8SwIrBbqksW\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"IlU35DSXGkesV\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"fl\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"1XJ1BKxdLr8xZ1\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"1OTUMDyCdYkVv8\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-session\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"Nuz4FnUt\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"HwzHAjBygsfiSUX\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"202\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"Yp4H7ZJmIH7Dv\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"6\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"EYhWYsmGpl4z0l3\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"8RFEMjEMC5stu\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"query\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"1ZkQEmh5ANn\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"0KsiqU7uz6ecQ\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Brain\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"nrOMFbiD09E\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"QnKfskbAhz0\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Fl\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"q6Uz9SRPWbTCf\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"ZFMpp0BC39uzCN\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" reasoning\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"2MwmX9\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" stream\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"2aJP5sGuM\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" instrumentation\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"obfuscation\":\"UvySj7uLIjgfiB\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"lookupId\\\":\\\"flue-session-2026\\\",\\\"query\\\":\\\"Braintrust Flue reasoning stream instrumentation\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"output_index\":0,\"sequence_number\":24}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"lookupId\\\":\\\"flue-session-2026\\\",\\\"query\\\":\\\"Braintrust Flue reasoning stream instrumentation\\\"}\",\"call_id\":\"call_fGesVCba2aABMoxSNfVnDVnR\",\"name\":\"web_search\"},\"output_index\":0,\"sequence_number\":25}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6070c48191b73ad8a8f9b550f7\",\"object\":\"response\",\"created_at\":1785748320,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748320,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_045458052dc10379016a705b60c92c81918da2fab9710ee199\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"lookupId\\\":\\\"flue-session-2026\\\",\\\"query\\\":\\\"Braintrust Flue reasoning stream instrumentation\\\"}\",\"call_id\":\"call_fGesVCba2aABMoxSNfVnDVnR\",\"name\":\"web_search\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1055,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":34,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1089},\"user\":null,\"metadata\":{}},\"sequence_number\":26}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432ba7c1a7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:00 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "236", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998488", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_b7babd52eb11438ca65da8e3e06cd733" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 2, - "id": "5d2312f425df9f2f", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:02.466Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"query\":\"flue instrumentation\"}", - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "id": "fc_045458052dc10379016a705b6008408191b27653755cf5bd03", - "name": "lookup", - "type": "function_call" - }, - { - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "output": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "function_call_output" - }, - { - "arguments": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\"}", - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "id": "fc_045458052dc10379016a705b60c92c81918da2fab9710ee199", - "name": "web_search", - "type": "function_call" - }, - { - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "output": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b611e8881919ad2adde8d6874b9\",\"object\":\"response\",\"created_at\":1785748321,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b611e8881919ad2adde8d6874b9\",\"object\":\"response\",\"created_at\":1785748321,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_4oiajYhDnktEd6wZwQjSMwpb\",\"name\":\"summarize_source\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"hK03W2coG6GFdN\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"url\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"Uu8U6Kh8Nj5Gs\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"sI2ly4ADW4764\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"https\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"ub8xQfxuyrk\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"://\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"JvDy4VPF3RT8t\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"example\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"nUx1jB9C1\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".test\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"nESgdh4F5D9\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"3SOSypdEqxfoT\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"WfNnPvGWaIcJYU\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/re\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"GSIAZIvM9SbTS\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ason\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"ELHGgjctvQHZ\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ing\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"AGLq1WDbPPiDU\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-stream\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"BuKw2YnJj\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"s\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"szEOAd66hgyQxiR\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"obfuscation\":\"nWuCGD2j8eK0lG\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"url\\\":\\\"https://example.test/flue/reasoning-streams\\\"}\",\"item_id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"output_index\":0,\"sequence_number\":18}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"url\\\":\\\"https://example.test/flue/reasoning-streams\\\"}\",\"call_id\":\"call_4oiajYhDnktEd6wZwQjSMwpb\",\"name\":\"summarize_source\"},\"output_index\":0,\"sequence_number\":19}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b611e8881919ad2adde8d6874b9\",\"object\":\"response\",\"created_at\":1785748321,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748321,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_045458052dc10379016a705b6194148191a2820dc720d327c5\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"url\\\":\\\"https://example.test/flue/reasoning-streams\\\"}\",\"call_id\":\"call_4oiajYhDnktEd6wZwQjSMwpb\",\"name\":\"summarize_source\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1147,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":30,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1177},\"user\":null,\"metadata\":{}},\"sequence_number\":20}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432be8c937bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:01 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "326", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998398", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_65060d10d7dc405b99f44cac4ba179c7" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 3, - "id": "83cbbf6da482c905", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:03.092Z", - "request": { - "body": { - "kind": "json", - "value": { - "include": ["reasoning.encrypted_content"], - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"query\":\"flue instrumentation\"}", - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "id": "fc_045458052dc10379016a705b6008408191b27653755cf5bd03", - "name": "lookup", - "type": "function_call" - }, - { - "call_id": "call_rBgonKxWHwhN9E8aQ4VobIOe", - "output": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "function_call_output" - }, - { - "arguments": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\"}", - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "id": "fc_045458052dc10379016a705b60c92c81918da2fab9710ee199", - "name": "web_search", - "type": "function_call" - }, - { - "call_id": "call_fGesVCba2aABMoxSNfVnDVnR", - "output": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "function_call_output" - }, - { - "arguments": "{\"url\":\"https://example.test/flue/reasoning-streams\"}", - "call_id": "call_4oiajYhDnktEd6wZwQjSMwpb", - "id": "fc_045458052dc10379016a705b6194148191a2820dc720d327c5", - "name": "summarize_source", - "type": "function_call" - }, - { - "call_id": "call_4oiajYhDnktEd6wZwQjSMwpb", - "output": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "low", - "summary": "auto" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Return a deterministic lookup result with an id needed by web_search.", - "name": "lookup", - "parameters": { - "properties": { - "query": { - "type": "string" - } - }, - "required": ["query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search a deterministic local web index. Requires the id returned by lookup.", - "name": "web_search", - "parameters": { - "properties": { - "lookupId": { - "type": "string" - }, - "query": { - "type": "string" - } - }, - "required": ["lookupId", "query"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Summarize the selected deterministic source after web_search returns a URL.", - "name": "summarize_source", - "parameters": { - "properties": { - "url": { - "type": "string" - } - }, - "required": ["url"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6286dc8191be79bd3493b23c38\",\"object\":\"response\",\"created_at\":1785748322,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6286dc8191be79bd3493b23c38\",\"object\":\"response\",\"created_at\":1785748322,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PROM\",\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"obfuscation\":\"ttkbAlDp50d6\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"PT\",\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"obfuscation\":\"Z6PXaim62hCyKh\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"obfuscation\":\"MiAXFIn3IuM\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"PROMPT_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PROMPT_DONE\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PROMPT_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_045458052dc10379016a705b6286dc8191be79bd3493b23c38\",\"object\":\"response\",\"created_at\":1785748322,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748323,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"PROMPT_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ma\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"low\",\"mode\":\"standard\",\"summary\":\"detailed\"},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Return a deterministic lookup result with an id needed by web_search.\",\"name\":\"lookup\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"query\"],\"properties\":{\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search a deterministic local web index. Requires the id returned by lookup.\",\"name\":\"web_search\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"lookupId\",\"query\"],\"properties\":{\"lookupId\":{\"type\":\"string\"},\"query\":{\"type\":\"string\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Summarize the selected deterministic source after web_search returns a URL.\",\"name\":\"summarize_source\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"url\"],\"properties\":{\"url\":{\"type\":\"string\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1222,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1229},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432c7682f7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:02 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "281", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998323", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_89fa79409a904f43a453a36d2ca890d0" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 4, - "id": "417e64addcf59da7", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:04.374Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b63235881a393625d6974243e44\",\"object\":\"response\",\"created_at\":1785748323,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b63235881a393625d6974243e44\",\"object\":\"response\",\"created_at\":1785748323,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_R2a5LOml0NZqq5XTcTCKfbMW\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"CbHdtJmEy68R82\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"description\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"N0n5J\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"oVS9emhElTbAe\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"44SOZV25Sx36v\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"5rEAHjD7s9AY\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" deterministic\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"XS\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"M9xvWA7f348PV\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Q6wOJG5PkJoZVd\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" instrumentation\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"YPWsSbID38RZ8M\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"DoKfuME0Em0z2CG\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"XBFbIY4Fow2p3wz\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" test\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"JX98m2RkrWD\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" skill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"yWTHn4IL3k\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tJChmRwpGkHi7f\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"59X6X9YSOr0tsbm\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"YX8fGT8PwVkyRUz\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"2vD42kigSA1dk\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"SR8lnZvAOFNlAd\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"e5LCrEk4SxTUxd\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"yQv5KXABVNyh\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".\\\",\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"3SJXRzWCr6dy\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"prompt\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"jnnoMIl9VV\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"cbOKoZJQXGHZv\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"lOwODhO09Mw2T\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"HJKMWHVzo3e8\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" skill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"pfSw7BP6rF\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" named\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"gdjMLdp6nl\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"hb129i9TWIbbM\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"EUDmWBk8oSvNHEU\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"PkeI32czJqwmlmr\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"H2UQSRf8sMZNRBG\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"coZw5rkSM9llJ\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"RTDrspHm2QXfY9\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"dhLU5GXYLswOhi\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"PNAPCNmx4QJp\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"CpbMQHjxstrcsu\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" with\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"KOoqv80ko3G\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" arguments\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"jcYH3O\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" {\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"szXf5lZHyCpilh\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"52sGZR8YVknFj3\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"marker\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"J8ZugJCeBJ\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\":\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"2Ffk7q5BGoV\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"SK\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"xeKflxURNDoVyb\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"XjKJ0bhJcewlH\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"R4FhmYdoiCO\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"X6XdGVk1CC6NjY\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}.\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"YYRBPkT64KLrlE\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Output\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tzFy24CSr\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" only\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"rf2kD3jJIfw\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"MZEAVTJ1ktEY\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" marker\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"hsFGNt1CH\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" and\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"7jo0OqDghkTU\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" no\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"x7Q4KF2sTB6xe\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" extra\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"5lWyL7VcaY\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" text\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"P6M5oge56Cp\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" when\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"aSZOArmUWCv\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" finished\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Bcwcv1j\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".\\\",\\\"\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"QPfZWJIaOZZK\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cwd\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"8CNMuVP6JkXXp\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Zyun5ONtg0VES4\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"/\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"0hQ1Tpvy40v6ov\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"AX7KPkq2wczcX\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"gqTRHNnYCqD98\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"gkSS5rmctsBm7v\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Mfu7ocBSyP\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"RYfnUiUEM7nSWVO\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"9TJqEA9kGKQ\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"lYlMUrHuRU3\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"7rcyz6emzCww\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"iuArNS0owWYS7ks\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"aO4xFHIWApy\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"MJOzy2JsM8I\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"sO3CIxyDjZko\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"NXKXqePfpHdojz\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"r2Ux15z\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Kab8AO35yKTKmS\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tIH5zoiqj9CTjyo\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"iLppUWbjIW0Alm6\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"eBh1cQtXPmFWjX\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"ejJv2wp49FgCgw\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Pe5APONBPL5Cz5\",\"output_index\":0,\"sequence_number\":84}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"8L98Ly29NFwgR8\",\"output_index\":0,\"sequence_number\":85}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"zhXouMXCcpBH\",\"output_index\":0,\"sequence_number\":86}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"tfHJ3cnyfuchLJx\",\"output_index\":0,\"sequence_number\":87}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"Dj1UH61VeDE9CVJ\",\"output_index\":0,\"sequence_number\":88}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"mEcW9oStLYq57Z\",\"output_index\":0,\"sequence_number\":89}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"da4hpEx6UsGmIdX\",\"output_index\":0,\"sequence_number\":90}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"wZYe5LO1qqwywa\",\"output_index\":0,\"sequence_number\":91}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"FPrjbypLHyxRn\",\"output_index\":0,\"sequence_number\":92}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"pj0NPDTch\",\"output_index\":0,\"sequence_number\":93}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"xd0ux5YOhgezS\",\"output_index\":0,\"sequence_number\":94}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"cETdOP2IPG6g1d\",\"output_index\":0,\"sequence_number\":95}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"MzOzX2St8G3JA\",\"output_index\":0,\"sequence_number\":96}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"TbtAamDa\",\"output_index\":0,\"sequence_number\":97}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"CJ8PGzQrtmy\",\"output_index\":0,\"sequence_number\":98}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"obfuscation\":\"wrvfq006IzH0mA\",\"output_index\":0,\"sequence_number\":99}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"description\\\":\\\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Output only the marker and no extra text when finished.\\\",\\\"cwd\\\":\\\".\\\"}\",\"item_id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"output_index\":0,\"sequence_number\":100}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Output only the marker and no extra text when finished.\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_R2a5LOml0NZqq5XTcTCKfbMW\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":101}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b63235881a393625d6974243e44\",\"object\":\"response\",\"created_at\":1785748323,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748324,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Output only the marker and no extra text when finished.\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_R2a5LOml0NZqq5XTcTCKfbMW\",\"name\":\"task\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":847,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":109,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":956},\"user\":null,\"metadata\":{}},\"sequence_number\":102}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432cb5fc97bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:03 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "206", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998698", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_4f8a543160de4adea11337a977a14713" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 5, - "id": "341675a0a1a88c63", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:05.801Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b648f7081a3b96733816812acb8\",\"object\":\"response\",\"created_at\":1785748324,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b648f7081a3b96733816812acb8\",\"object\":\"response\",\"created_at\":1785748324,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_w6pyYZgnhlSOAXul75DAG1Cj\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ljpLbllVFarTqw\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"description\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dI5yK\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"gaE80vRkyue6q\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1yh2B3GwgUIkU\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Aoc2XDyvRflwpK\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"gKDO3Feq1OjvKMT\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"d2O2IlHGALtgxiz\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"c9m5GXjLlVm1X\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"NUY5MBuNzz8sZQ\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"9Pwv64hhPdkCsg\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"iOoKz15nYfXp\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" with\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"UCahcEprOGt\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"TaOkeLqZF\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" SK\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"VxlrD2puv3ABI\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"DebTGrlabFRQY\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Y6K3aF28EHz\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"py2Pd0lnhbcOC\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"prompt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"PM2ySL1u5c\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"GEFeUIpfX9F6O\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"g8vFm9KNQk3gN\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"cXxSzWyVKRHs\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" skill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bwbVemN5vW\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" named\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"4DdymViVIi\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"9QwzJKU8yuXI1\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"qc8c9FjKJwS85NJ\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ewNZxFO4TdAgDUM\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Y8xyYUu8EtAbcT1\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"88SoGMEyRHz8M\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"DpP2H8WkpDUzag\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"5jctq9RddqcIt8\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"xpmcaPHyYGYa\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"kumygUHU0Bd682\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" with\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1aD8huNzRAg\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" arguments\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"u62pY1\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" {\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"DlufXJUKJUw7nw\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dqIkwjYJ3Ruj51\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"stUcL5JVqc\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\":\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"cUfPJq2cus1\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"SK\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ZzPWTx2aFPwBYM\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Hbw389fTXAJOT\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ALgrsM7YeM5\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\\\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"D4fcykCofi3ap6\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"79o07UPm9B29cw\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" Ensure\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1JXF9XGzd\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" completion\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"vfueK\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" and\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"VlKAzFGFtrcw\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" report\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"wBeH1rhXr\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" the\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"6EO4vypCMr7q\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" final\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"0pF35NFSxj\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" output\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Fyh51oBFK\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"2fdE8CEMT\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" only\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Oi1kYTafLLr\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JHNARwrOOg28\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agent\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JujpXurFNFJ\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"7UtnbpjQ9Vd5r\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"lUMgv4pWgFguMgN\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"lquqyIOhUG8FFqb\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"z1aAXV10WTtEwfO\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"z29NnUuO8awvU\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"vjUb5xZd4tz6Nc\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"S79uypIplW7mcH\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"GgbDCJps9iY5\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"vCCHCimmRyAgx\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cwd\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"AnLDQiNay8ENp\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"6YVXzJiSMPKoqe\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"2iVYgZDItlXYju\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"p8iffYf14P4rP\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"jHIrIgM5myBm6\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"Yud9IuaLw1CQTi\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"sxVNzIOgmI\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"BFftBcl4q4mRiic\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bvKWRZsGF3z\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"8fCBYfWjld6\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bWCV0wB1yOIj\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"gMLQ5nGNZU5RQcA\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"HbDV4gJR67J\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"fY6Fye7lRfU\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"nDcCSPqarvNr\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"CDVkHgmi29Aam7\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"yMH6cPx\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"zoEfWKIxJ3vi4r\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"ti33GAqnm3z55of\",\"output_index\":0,\"sequence_number\":84}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"iKabCz0geb00km7\",\"output_index\":0,\"sequence_number\":85}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"kQaYnx9LpaZ4YA\",\"output_index\":0,\"sequence_number\":86}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"rHzMr4AWuPUfim\",\"output_index\":0,\"sequence_number\":87}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"nJma8KAPeFozgD\",\"output_index\":0,\"sequence_number\":88}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"bJV57iCDmqv2VA\",\"output_index\":0,\"sequence_number\":89}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"IpVQRAjXMPE4\",\"output_index\":0,\"sequence_number\":90}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"pzJyXElP7ou0ovn\",\"output_index\":0,\"sequence_number\":91}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"1ZlIYGuEL0CGzns\",\"output_index\":0,\"sequence_number\":92}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"QVS1D8jJbqlZ7X\",\"output_index\":0,\"sequence_number\":93}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"2mhaJD9do70hkwJ\",\"output_index\":0,\"sequence_number\":94}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dXooQVlSHUeRZP\",\"output_index\":0,\"sequence_number\":95}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JYgibM5tWMAjH\",\"output_index\":0,\"sequence_number\":96}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"JH3lHc5hu\",\"output_index\":0,\"sequence_number\":97}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"S7XC3UsMmTfuf\",\"output_index\":0,\"sequence_number\":98}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"xWgqKM1XAI7Gka\",\"output_index\":0,\"sequence_number\":99}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"7l79NEJnKe3pM\",\"output_index\":0,\"sequence_number\":100}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"F9vfZGnW\",\"output_index\":0,\"sequence_number\":101}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"iNKeVMoTNBN\",\"output_index\":0,\"sequence_number\":102}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"obfuscation\":\"dJaOqvNAHb8EH5\",\"output_index\":0,\"sequence_number\":103}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"description\\\":\\\"Run e2e-flue-skill with marker SKILL_DONE\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Ensure completion and report the final output marker only.\\\",\\\"agent\\\":\\\"e2e-flue-skill\\\",\\\"cwd\\\":\\\".\\\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"output_index\":0,\"sequence_number\":104}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run e2e-flue-skill with marker SKILL_DONE\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Ensure completion and report the final output marker only.\\\",\\\"agent\\\":\\\"e2e-flue-skill\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_w6pyYZgnhlSOAXul75DAG1Cj\",\"name\":\"task\"},\"output_index\":0,\"sequence_number\":105}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b648f7081a3b96733816812acb8\",\"object\":\"response\",\"created_at\":1785748324,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748325,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"description\\\":\\\"Run e2e-flue-skill with marker SKILL_DONE\\\",\\\"prompt\\\":\\\"Run the skill named \\\\\\\"e2e-flue-skill\\\\\\\" with arguments {\\\\\\\"marker\\\\\\\":\\\\\\\"SKILL_DONE\\\\\\\"}. Ensure completion and report the final output marker only.\\\",\\\"agent\\\":\\\"e2e-flue-skill\\\",\\\"cwd\\\":\\\".\\\"}\",\"call_id\":\"call_w6pyYZgnhlSOAXul75DAG1Cj\",\"name\":\"task\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":855,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":113,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":968},\"user\":null,\"metadata\":{}},\"sequence_number\":106}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432d37fd37bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:04 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "268", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998692", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_bf6f5e02b3da40c19ef5905452f8e5f6" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 6, - "id": "cbb9070d68f84d1b", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:06.856Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b65dc0881a3a79d930ad7069d95\",\"object\":\"response\",\"created_at\":1785748325,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b65dc0881a3a79d930ad7069d95\",\"object\":\"response\",\"created_at\":1785748325,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_6zEBnJRmXFIkGFwNjWLZP4uC\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"qYr2RE37LRYj8T\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"5olycNCp4\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"TtIA15U6WTwyB\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ls\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"8gpmFqa6OBiMdp\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"HqbfEOyA6n1XM5\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"la\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"zkZ09ufXuvTuYH\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"IsgyMPXjN3RyQT\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"QwfPCO1ymGdY7\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ktEAC1neLBJD0\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"qSxXdSs4TsmFt5\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"NIrFRZbher\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"h1HKJqOxjX7neu9\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"g5de6pXeQeu\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"O8W7X6T6QJ9\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ZwdOUP0nNaaU\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"RdDtnAGnoUWZHMl\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"bJadnE3z1Pn\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"QdtHJab2Nju\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"d0BalsPUN30E\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"zkZSKvyjjnJqmd\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"gidR8ss\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"sTij2BeE8hhf4F\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"U6tRlf8eVA0ZfFt\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"3iym56BtyHXWWzj\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"yPazRePg1CYCjy\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"XlYRyV5eP5RuAi\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"cEUjV7aq8IJmrd\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"llpLkANV2zyfnb\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"flFQ8Wd13kY2\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"6NV0XhXU74sXecB\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"3e8DIPv3Q4zimTJ\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"CcYFgrShio2OGa\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"SB8CftjcqKYqDAq\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"o0TbOyqaOwnbjc\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"AxgExC2R2UrOj\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"GktP5dj47\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"D8Qyuf0OZ5udR\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"rTVImQr26P3Vsq\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"SPXiRAIfqk20q\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ukKgXlCi\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"q5sV1H9oEPR\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"Vbdq3gYacR4ydn\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"iYjX4AJgNl\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"QpXEqJPwcIzx8\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"idAm4VrdUd9\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"xlH3WH5e7K7TVT\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ci7N2JhvoR4v3\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"lNESXh61G4kcn\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"JHLNRMdy9opWYh\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"dgKik8Wq8j\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ivSoP1Az4lunbMK\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"ei4Les3EoOx\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"j583CvShsMn\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"8pCfiAeFxnGs\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"BrQSS7SwzKOQJMI\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"mq3EFmYB311\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"hO6jOpGPatJ\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"lbKGYWB3E1BM\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"u16CBhAKfJTH5L\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"5hJroXc\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"33pMuKaTNDmMqZ\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"7PfD7iSXELGdvpH\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"l89ZUwAGJ4COOHT\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"heNzYKqh1OCQTD\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"sU1Q9AhPAADW3\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"G2KErWebcGP\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"Hk3AzR5adCxbnFT\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"4\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"2AwssIctaSpQjDt\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"qTL2wxyksScrpT\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"9dAiG7pNttSR\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" AG\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"vyWvpkSniB10E\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ENTS\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"8X34qqxb7k4T\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"srlvyGZKNg8fy\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"kX3C3ee4M4LAQZ\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"kuuWd4dbfED\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"t1AZzeTRFs1dW\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"J9nNvn2yP\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"HoGIKlERH8SubV\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"K0vUuFSWy0H29\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"iki9PdBtWpJFT\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"obfuscation\":\"XUhEnBL2ma1rfYy\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -name AGENTS.md -print\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"output_index\":0,\"sequence_number\":84}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -name AGENTS.md -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_6zEBnJRmXFIkGFwNjWLZP4uC\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":85}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b65dc0881a3a79d930ad7069d95\",\"object\":\"response\",\"created_at\":1785748325,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748326,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -name AGENTS.md -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_6zEBnJRmXFIkGFwNjWLZP4uC\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1002,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":94,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1096},\"user\":null,\"metadata\":{}},\"sequence_number\":86}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432dc5a4c7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:06 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "201", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998545", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_3ec73f40ef984f3ea2f7b4df76b2890e" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 7, - "id": "bccc54d212e2503d", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:08.056Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b66ec3081a3b53487cd28a912dd\",\"object\":\"response\",\"created_at\":1785748326,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b66ec3081a3b53487cd28a912dd\",\"object\":\"response\",\"created_at\":1785748326,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_jQUCie0ZgaXUrZCDcezp0msl\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"5s5fPUr1lnx6LN\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"EzIXsl5P7\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"rfpcd360vn0Zc\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"gbSugqPh322J\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"un8JuSPgzNW7Tq\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"AT24QvJgyIKCb\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"M2WqPhiDm0W0B\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"HnCl5JHASkkvRV\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9hqImMoIYc\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"G2MFkp35u1CeVM3\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"S9DNif6AAaj\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"KFRgE1qwn8O\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"iRZhcqXd9XRs\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9XF5P57pZo3uNAy\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"4P9IWtruXv8\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"6tLj1KQL0OS\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"ZJX7xdMeL6wZ\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LJ7CGVyndyKtYf\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"xJcnaku\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"PfTOXv6id59uZk\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"ZygymPXfo8RsWR7\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"tcZjEPZR3q54QCO\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Is3Ft8kklGHlSC\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"bt\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"J9UUPkfaryOCWA\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-t\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"tUpNGsjI3owh2O\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"mp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"uhvu4t3Z33cBrQ\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/run\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"EY7RAwkNVqHj\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"WzBJvCmbkvTjzRe\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"9\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"T1c8ikRVDyXs3K4\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"sp\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"TLa4CtuK49G4fL\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"u\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"vUKKjMsXw5sIkHv\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"Hg\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"SD4uFMMoVX66s1\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"dsOEaf4EkLy14\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Sk1iL5lxc\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"j3F97pquP3wT0\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"5SHqC6FW1engeD\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"MKvWbeQhFJuyU\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"0JPR573Y\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"tJ0iHGcFnz2\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"2Xe2zCjtFpD1Aa\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9fKJvO4eySsc7\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"adTuj16Pog1\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"RG1GXpp0dtqROaL\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"4\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"62nieO7lhOujIMV\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"f85mHnnTvDNVsm\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"oOkDwdNirXZn\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"7hsgm1tQLQmfWU\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"TEtJIF7zjvh5BC\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"TRWuvVnlxjuw\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" AG\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"OPMH46rRaY9wj\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ENTS\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Wx1ktH8noul5\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Oo8KevjbHLtSH\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"W1tKVt07sDFz6L\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Ca0wTzfowmG\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"InAeZlZgCo2EWg\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"exec\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"xuVbI7qC8ZR7\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" sed\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"rWSvxjZ31UQg\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LsdCqDwIUZ1aXY\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"n\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"9t5AteQjutiyqe3\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"b8zSZflAk9EXLf\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"1\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"i7Uvk2jgzfKhd4j\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\",\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"dY2J1TUAF8vRFo8\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"200\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"3t3yMCd2EF2u3\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"p\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LI717GeLz7d3rCb\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"aUymdN14yxm7RVU\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" {}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"SsZA4kHNYtJqo\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \\\\\\\\\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"VZIldvECynY3p\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\";\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"LXiCXRoQhZXP5HK\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"lN4m4dNvIh4ID\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"bGxGZD252\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Zz2e8ECX6n6SVc\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"Zi6V6iOPLR6s7\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"rp63i4YuRxltd\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"obfuscation\":\"74rLY4yKHw7IjTn\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\\\\\;\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"output_index\":0,\"sequence_number\":77}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\\\\\;\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_jQUCie0ZgaXUrZCDcezp0msl\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":78}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b66ec3081a3b53487cd28a912dd\",\"object\":\"response\",\"created_at\":1785748326,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748328,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\\\\\;\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_jQUCie0ZgaXUrZCDcezp0msl\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1197,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":87,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1284},\"user\":null,\"metadata\":{}},\"sequence_number\":79}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432e308c57bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:07 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "181", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998347", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_5c4fd875329344cfa3d20c5925805429" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 8, - "id": "8eec740c1be01c5c", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:09.317Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6820c481a3b64f6527345dc3ca\",\"object\":\"response\",\"created_at\":1785748328,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6820c481a3b64f6527345dc3ca\",\"object\":\"response\",\"created_at\":1785748328,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_NgMSUb6wIR0GAf9U8LXERtBQ\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"Ecm4AJUnR33nzy\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"qAhzrPXMU\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"HjoLYpLsOOApn\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"COx5RXyau1iu\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"7fcdZxijyLBFbj\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"EdhPBWD97UXXj\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"g4JtDEv0GyNkp\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"Yu3ytbv7EfB9L7\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"loDk0mviWC\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"4KyvKX7bONkWif0\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"yowGulOqnpo\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"83qIG2PGqLt\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"ZFbal7RnszIt\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"GVlOdY86Xo86jAw\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"mjcerP27QOt\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"oQnyazxVeT6\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"2CRHfaUuS1Dd\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"3IfHRkUmRfpbWX\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"uJPFidS\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"PAbcespQ5gkoQu\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"By2Tx9QLwPZRKd1\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"jCZWJ7sTG0eCEsh\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"gu7kgzv2IJ1Nyl\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"ftZoJcWqYeSjP\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"pVAaXJ9DEpZ\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"sR3ARyMb8XihmRv\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"6\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"O5oruVniSAltaiT\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"vv0l4nT2EtqHMG\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"eCQV3vrXZ37f\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"LD2tChTMhpkm7O\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"dbjx8JTI5wqqpk\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"CMGMxsC84MqB\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"IP4ilgz1kRzQlU\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"AG\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"30S1wp0FSdH8j0\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ENTS\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"LbBUDVVtGwst\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"8piB3U9Zy7jyA\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"9DRBJbRy6maC4gS\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"XAt6MDPojUNwz3\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"2sb7d6gE12q\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"78xkqC87iDA2l\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"LZICbxiFz\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"L39ZgCy9F9L7ou\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"cmBrFOna5ifiL\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"z5qEzhvJIbUVN\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"obfuscation\":\"S29PMSSd6WVTIYN\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"output_index\":0,\"sequence_number\":48}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_NgMSUb6wIR0GAf9U8LXERtBQ\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":49}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6820c481a3b64f6527345dc3ca\",\"object\":\"response\",\"created_at\":1785748328,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748329,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_NgMSUb6wIR0GAf9U8LXERtBQ\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1298,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":58,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1356},\"user\":null,\"metadata\":{}},\"sequence_number\":50}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432ea7f5d7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:08 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "428", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998245", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_e00fc12bdba24599905386f92e325433" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 9, - "id": "6945bf18d345ad4a", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:10.503Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b69681081a381f7cdcf2e5f8c38\",\"object\":\"response\",\"created_at\":1785748329,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b69681081a381f7cdcf2e5f8c38\",\"object\":\"response\",\"created_at\":1785748329,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_BXKiQ5KG8c42XPPp94zgcY3H\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"hG7XwE8kTftCFD\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"SAESyTCDU\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"FdSHAU6VPfEiP\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"JTitwOdHc00P\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Z5ba86g8nOxzKr\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"BUYeIHkrw1ca3\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Jnlz6e6YIi8An\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"EiAxOlmPQ6R9Lk\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"HLwMNKLqXd\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"CyMa169D7RtZmdr\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"eBBwyrijOzo\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"9oZJd40vJbv\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"au0nu8FGh24U\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"mH2gvFHGB1gaA18\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"W1iZL4cRSMq\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QUUnH5ISGOi\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"eeLPcF9XcrHV\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"lJ7n3mjPmhDNxo\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Cq8la1K\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QCMTFUK5pMEVRm\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"BgRGVixu3UbxQDA\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"aqbUU7TCMgXGNus\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"0aahbmWnQc34bI\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"hLd8UqgGfFZ9D\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"GmhwY6EWAHh\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"NRuEQgh0p3eRHWo\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"5\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"VSUdF5XfszF7J7C\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"7bgbWInYXsOzEZ\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"ZdXd8MYxzBVa\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"lKHlgmcHv4yQ7n\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"O3v0EDtKmgDYYi\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"yOsJv2sjpuJT\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"bT4zdeKIYUfHc\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"jujglZGbsBwPa7\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"LKKBJiCjnXh4xf\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"shy3qPjXsnqPzfn\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QCXNC1UuV7e\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"XG8mVOll3Ohqe18\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"twOToqPAOsKMMVy\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"DZtAspDqcPPg83\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"o\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"1vqyG924TjjeMm1\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"vbMfw1I6yQT071\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"name\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"yEkTfUjLa7T3\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" '*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"QbcNOxt5ouCjy\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"fkrT1lk0jv9\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"*\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"9RVtGG8sxWEOjJr\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"'\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Za4XJQ2Z2IsOIeb\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" |\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"px85R0MdttFruZ\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" head\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Y88f7Ld1Ora\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"8moCEbjvgTG1n\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"Kmyh4pQvd\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"GrJ0KmP1JEa4oC\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"tQ7b2AdVP1fwn\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"OGHfUzdqw3zns\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"obfuscation\":\"4USiuoYvOOTVjiY\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"output_index\":0,\"sequence_number\":58}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_BXKiQ5KG8c42XPPp94zgcY3H\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":59}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b69681081a381f7cdcf2e5f8c38\",\"object\":\"response\",\"created_at\":1785748329,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748330,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_BXKiQ5KG8c42XPPp94zgcY3H\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1370,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":68,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1438},\"user\":null,\"metadata\":{}},\"sequence_number\":60}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432f26f707bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:09 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "283", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998176", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_f5479925a7844533a4ae27ed832d027b" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 10, - "id": "ab34eb26595033a1", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:12.905Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6af7d081a396a76851300752a1\",\"object\":\"response\",\"created_at\":1785748331,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6af7d081a396a76851300752a1\",\"object\":\"response\",\"created_at\":1785748331,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_Zmmrs64LFscpAHsIUaIfEcAN\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"K1hVgurfEtUc37\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"luCO3JGp3\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"FmitdgZUwsXVs\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ls\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"VZpJ9At0woeytk\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"0xukceLNBwYJiK\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"la\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"1s2AnPNungEZlQ\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Bfcx9jEBxq6eit\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"1rFsvnpXQsSA2\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WHxut9fzLrXkW\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BzqeyKc2WZeyRg\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ii7jPgkIJ8\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"PkykxLdvlfrkHPL\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"12UfuwAycuU\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"3K3OjXe08nc\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"PkaaeYRGj55a\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"UHfCbDbKjO5lYqr\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"MOTWOSsU43Z\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"dmMQk0sQrGh\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"XtEiI6ADgEIV\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"5ovKtom8zqryPR\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"7eUUx0L\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"9QbEfSAXVpmE07\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BeGJDWOdDgtyZwk\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Vio65vRQsUs4hgC\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"6P7cD2KzvBQE3\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"RJSR2IM5S\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"uSAAuJhHqx2h6\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"LURBTZTvJSTf7s\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"b3ZtE4ekxYWXs\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WUoTVOo4\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"n6TvbFOtdSH\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"1Lt96k7JA9dUeI\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"O7qjKmlEms\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"I1E6t9owIg9P1WR\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skills\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"21ZpKkvrdb\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"zV3b4fO0R5USoJ\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"wtfZeZvShndVNQM\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"8DgrYaTc5tF9qQN\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WMhoTEOnE6Xro\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"XaFg3AblnqowIQ\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"FJ1AtHWIUTZkuK\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"K2KXxbd3uIZt\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ek6YUPE8Cp9AE\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" find\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"XM049MlPwBE\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"SCFb56qc5ogbCE\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"nB5JRT4lhiSpZ\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"DozjlarmrsGSB\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"CvaFfLoJcDammS\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"6yYxC29jBg\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"NpUUfvLkXCyoKMW\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ym9bVn0oRVM\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BPomQliEf6i\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"0hcPCne66Cg8\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"TfdWF7zbYNainuF\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"eYEUL1yBJZh\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"EfjIxAAX6o0\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"8hFJ6Wya2ZL8\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BUmLaB9uO7p4JA\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"TZCxxmh\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Wkjm95wXRvIPMz\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"IWO2TynZ5pN8CwA\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"AYKtiZZuh0gJgYj\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"rcDuq1t53ub50\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"cPtht6oAy\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"tQudFftzB650V\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"3iUp76L3SIYhqb\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"ZXAa0ULdLud1i\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"kzf0iJxr\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"rw7Suy9RUTq\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"uPuc49BoOe4rwv\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"xCcbSL9wCG\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"0kPSap2qJXeYe5c\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skills\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"07SeHSKAgE\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"g2UQCFv27V3sqv\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"YniEsIzqh6veIWP\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"64iQG5gdZWPuPkQ\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Kz4PY7cfevVr9\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"dAurB55pjAoLFj\",\"output_index\":0,\"sequence_number\":80}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"o5PWW9PAQB5Kvq\",\"output_index\":0,\"sequence_number\":81}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"kEfcPgdMqlKI\",\"output_index\":0,\"sequence_number\":82}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"SJWGXR6ryAJ2DE\",\"output_index\":0,\"sequence_number\":83}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"max\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"4QU605ENItN8B\",\"output_index\":0,\"sequence_number\":84}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"depth\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"4XthyjdZM9w\",\"output_index\":0,\"sequence_number\":85}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"WgGDumDYzUufFUI\",\"output_index\":0,\"sequence_number\":86}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"oQ8wQwbVorceatB\",\"output_index\":0,\"sequence_number\":87}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"sxrOiShGGRNzfx\",\"output_index\":0,\"sequence_number\":88}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"type\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"BwSNPMU6iGnd\",\"output_index\":0,\"sequence_number\":89}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" f\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Rm35RryQU13YdR\",\"output_index\":0,\"sequence_number\":90}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"zJ5ddKoSwCzVU4\",\"output_index\":0,\"sequence_number\":91}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"print\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"h80D5JBABPH\",\"output_index\":0,\"sequence_number\":92}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"rpdDODEBJorbc\",\"output_index\":0,\"sequence_number\":93}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"p4vKlv5m6\",\"output_index\":0,\"sequence_number\":94}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"Zbhx8YnzpAicDP\",\"output_index\":0,\"sequence_number\":95}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"r4y6kurWe2zky\",\"output_index\":0,\"sequence_number\":96}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"x0tuUliZpjxkA\",\"output_index\":0,\"sequence_number\":97}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"obfuscation\":\"VTLJl2mMhQOUOyQ\",\"output_index\":0,\"sequence_number\":98}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"output_index\":0,\"sequence_number\":99}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_Zmmrs64LFscpAHsIUaIfEcAN\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":100}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6af7d081a396a76851300752a1\",\"object\":\"response\",\"created_at\":1785748331,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748332,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_Zmmrs64LFscpAHsIUaIfEcAN\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1513,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":109,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1622},\"user\":null,\"metadata\":{}},\"sequence_number\":101}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25432f9bf6f7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:11 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "688", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998032", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_2ceebd83455e4b27b79fc2d72fb5dd3c" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 11, - "id": "250e64f864c72337", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:14.154Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6d008c81a3a332b5376350158b\",\"object\":\"response\",\"created_at\":1785748333,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6d008c81a3a332b5376350158b\",\"object\":\"response\",\"created_at\":1785748333,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_KXcpmIy4Cbfugk4SNT2Ge7sG\",\"name\":\"read\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"AplT1bUEIbLxZO\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"path\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"m2vm6uJH1fpp\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"UE4B3xJGoZjFiD\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"vXAYv4q4OGZeBm\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"lZr7JtLSOpK81\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"AlOhsp6mt6WsL\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"5nqW1Lb0h9tswT\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"A7di1P36tJ\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"sXrATqCJ4vWOD1I\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"dOlnaweYHcH\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"1e9JdY1CZrh\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"ItoYfCrGd190\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"U2GDfehNMcQdTxK\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"3sMRftpfAWB\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"RtP3uejwoaW\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"WTDkaRqKEdlO\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Sicm3wGutAJOUR\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"i3Iuw9L\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"cNyMn1ymq3OidT\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"TF4UthkOjNccn7Q\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"WCH0Nr5syDCGWhy\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"9QxeDK80jR88i\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"ZWEwRV5Aa\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"GCLbXJEBfam0u\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"fYviPD7BD1YoJ7\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"12G2xIB4r4pGx\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"RBo7r0Kw\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"LdN0iq6VGNg\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/.\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"SC3tJji0aA9vxI\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"agents\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"46FVinGMH1\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Tu4YeGAIlzJv7RO\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"skills\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"chyXB4Vg5n\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"koR43IjON9fzgj\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"2PamefwKxxhd235\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"7OGNeIaOwukKH4k\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"LyzjvJ7oVbrNO\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"F6FvzwFZ7LJK05\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"PvR9f24G2kgLxp\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"kill\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Cc4f9PrTB9gy\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/S\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"NFmNKyCIgJRpgX\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"K\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"i6Hkkg289FwqvEa\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Z0w7G2ePBPqRw\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".md\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"U8sCPOgqSHwmL\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Rj1PgiaJnVY9e\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"offset\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"Nv7VtoPO8D\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"xMA7iDRhGXyoRS\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"1\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"4HmQwBvniK7325Y\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"fp7VCNlkmETdHR\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"limit\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"JnAHVomsUGV\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"oMhJTWBoaVQje5\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"200\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"pck09M2RCSfdF\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"obfuscation\":\"v0WZruprCkbMb5J\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"path\\\":\\\"./.agents/skills/e2e-flue-skill/SKILL.md\\\",\\\"offset\\\":1,\\\"limit\\\":200}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"output_index\":0,\"sequence_number\":55}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"path\\\":\\\"./.agents/skills/e2e-flue-skill/SKILL.md\\\",\\\"offset\\\":1,\\\"limit\\\":200}\",\"call_id\":\"call_KXcpmIy4Cbfugk4SNT2Ge7sG\",\"name\":\"read\"},\"output_index\":0,\"sequence_number\":56}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6d008c81a3a332b5376350158b\",\"object\":\"response\",\"created_at\":1785748333,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748334,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"path\\\":\\\"./.agents/skills/e2e-flue-skill/SKILL.md\\\",\\\"offset\\\":1,\\\"limit\\\":200}\",\"call_id\":\"call_KXcpmIy4Cbfugk4SNT2Ge7sG\",\"name\":\"read\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1762,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":1536},\"output_tokens\":64,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1826},\"user\":null,\"metadata\":{}},\"sequence_number\":57}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a2543308de867bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:13 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "248", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179997783", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_33374d00629445b4aa73562f25cf18f0" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 12, - "id": "fecb7c9f8d04e1bd", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:15.213Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - }, - { - "arguments": "{\"path\":\"./.agents/skills/e2e-flue-skill/SKILL.md\",\"offset\":1,\"limit\":200}", - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "id": "fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997", - "name": "read", - "type": "function_call" - }, - { - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "output": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6e34cc81a383184de2af428d09\",\"object\":\"response\",\"created_at\":1785748334,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6e34cc81a383184de2af428d09\",\"object\":\"response\",\"created_at\":1785748334,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_xQybhnQYuflmfp7gyAScVKjY\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"IsfKDqHlaKpPlc\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"05wTcU2vE\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"I7n2P9cIUaGZ5\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"node\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"66No8n2S3lQV\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"pfQfFqmrd7Cttd\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"v\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"MsjXV30TNBXaVIN\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"sbVmMrYzWCmWN\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" ls\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"f3AOiA6bYx0WA\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" -\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"YwMIVfxBwLPyPz\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"la\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"yosVAZ9d354t0E\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"RtzYTt0vUd1KL5\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"UZCqFJvIdZJfi\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"ob8WdL1Zrd6z4\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"SHFL6SDvDru2gO\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"SlANW27QBQ\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"Llwwd75yCVN6cFY\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"Sp2PfwThtXK\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"WTr9EL0CMfU\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"iXd0y36BKeMz\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"TMVJ4avBWNU1PMg\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"gGYEjs7floV\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"nJfWpNMEeYh\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"VnnFujipwf0B\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"AAXM8t27HpiY8a\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"51XrioX\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"zAaUUwSEMO3Bxs\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"vyBpS3fu1undfoT\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"myDkZQqFtQPgErv\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"7jYdsKtAqGJsx\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"FgfnvD0oV\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"ECM3KkIZxcc2j\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"XIzDZYr4JpAjhV\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"VudZPLnjNH5PJ\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"i2iTyJpP\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"ZtblFC8j0Wk\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"1l1tQ40NrGj8E\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" cat\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"41U8EU6Ib9cg\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"cglsLI76yH9ClQ\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"qvk54uq2LZnZa\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"aovMJj1uFOFEM\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"NQpTAggULtv8zO\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5837PUGg7V\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"JhMbsWIyEoPXZRB\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"wUKGqlUZBXL\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5fqXYCtg0Gt\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"4Q1cI53ucyWL\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"abd5F6mWTBD3FqR\",\"output_index\":0,\"sequence_number\":49}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"YwaHJecYa6Z\",\"output_index\":0,\"sequence_number\":50}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"IZiyFBgwGNk\",\"output_index\":0,\"sequence_number\":51}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"DefferqgDs1G\",\"output_index\":0,\"sequence_number\":52}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"jf9cr3akDjVA60\",\"output_index\":0,\"sequence_number\":53}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"XaGvBNu\",\"output_index\":0,\"sequence_number\":54}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5UiGHyP1PGgpsP\",\"output_index\":0,\"sequence_number\":55}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"WKaJqwMQWdtBVYa\",\"output_index\":0,\"sequence_number\":56}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"DiOuTdnwYCCUpLq\",\"output_index\":0,\"sequence_number\":57}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"JNIox4EYrQqid\",\"output_index\":0,\"sequence_number\":58}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"rQexF5fiq\",\"output_index\":0,\"sequence_number\":59}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"CxjB3QQkwwxPq\",\"output_index\":0,\"sequence_number\":60}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"KvqTdaudMieVrZ\",\"output_index\":0,\"sequence_number\":61}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"hqrEuju9Hubvj\",\"output_index\":0,\"sequence_number\":62}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"xqdV3tk5\",\"output_index\":0,\"sequence_number\":63}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"finNWbOcOSK\",\"output_index\":0,\"sequence_number\":64}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/package\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"w5O538dR\",\"output_index\":0,\"sequence_number\":65}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".json\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"5lixmLK9osS\",\"output_index\":0,\"sequence_number\":66}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" \",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"UfcwAy15pK3UyDX\",\"output_index\":0,\"sequence_number\":67}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"8kuEminD3O9nEe6\",\"output_index\":0,\"sequence_number\":68}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\">/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"0ujGWvOIOaHg9t\",\"output_index\":0,\"sequence_number\":69}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"dev\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"l87mZp3YVC56A\",\"output_index\":0,\"sequence_number\":70}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/null\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"z1OHwKMfyTN\",\"output_index\":0,\"sequence_number\":71}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" ||\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"Nd0PVbdoXgVdP\",\"output_index\":0,\"sequence_number\":72}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" true\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"gYO8swE9D00\",\"output_index\":0,\"sequence_number\":73}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"dAlT06VEGUSd4\",\"output_index\":0,\"sequence_number\":74}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"9zDDLkKve\",\"output_index\":0,\"sequence_number\":75}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"bOgN2cbK9QPYUV\",\"output_index\":0,\"sequence_number\":76}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"2ycAaEUguB849\",\"output_index\":0,\"sequence_number\":77}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"jCkJ68Rcb58dt\",\"output_index\":0,\"sequence_number\":78}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"obfuscation\":\"bbF8wOnG3Zvtgvs\",\"output_index\":0,\"sequence_number\":79}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"cat ./package.json\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"output_index\":0,\"sequence_number\":80}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"cat ./package.json\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_xQybhnQYuflmfp7gyAScVKjY\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":81}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6e34cc81a383184de2af428d09\",\"object\":\"response\",\"created_at\":1785748334,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748335,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"cat ./package.json\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_xQybhnQYuflmfp7gyAScVKjY\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":1882,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":1536},\"output_tokens\":90,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":1972},\"user\":null,\"metadata\":{}},\"sequence_number\":82}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25433108e2d7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:14 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "170", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179997663", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_be4542a23d5f4e17b4285d598ac2400b" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 13, - "id": "eebc152ba05b8412", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:16.175Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - }, - { - "arguments": "{\"path\":\"./.agents/skills/e2e-flue-skill/SKILL.md\",\"offset\":1,\"limit\":200}", - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "id": "fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997", - "name": "read", - "type": "function_call" - }, - { - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "output": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"cat ./package.json\",\"timeout\":100000}", - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "id": "fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "output": "v24.14.1\ntotal 244\ndrwxr-xr-x 7 vercel-sandbox vercel-sandbox 4096 Aug 3 09:06 .\ndrwxr-xr-x 46 vercel-sandbox vercel-sandbox 4096 Aug 3 07:55 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 07:55 .agents\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 38 Aug 3 07:55 .flue\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 218 Aug 3 09:11 __cassettes__\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 4096 Aug 3 08:49 __snapshots__\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 12484 Aug 3 08:45 assertions.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 380 Aug 3 07:55 cassette-filter.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 215 Aug 3 07:55 constants.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2000 Aug 3 09:06 package.json\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 167098 Aug 3 09:06 pnpm-lock.yaml\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 123 Aug 3 07:55 scenario.cli.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 6877 Aug 3 09:06 scenario.impl.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 203 Aug 3 07:55 scenario.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 8066 Aug 3 09:09 scenario.test.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 121 Aug 3 07:55 scenario.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2376 Aug 3 08:49 scenario.v2.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 4616 Aug 3 08:50 v2-assertions.ts\ndrwxr-xr-x 4 vercel-sandbox vercel-sandbox 26 Aug 3 08:06 versions\n{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.4\": \"^3.1.4\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.8\": \"^5.0.8\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\"\n }\n }\n}", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6f493c81a3a225e3a6b86bbee9\",\"object\":\"response\",\"created_at\":1785748335,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6f493c81a3a225e3a6b86bbee9\",\"object\":\"response\",\"created_at\":1785748335,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"type\":\"function_call\",\"status\":\"in_progress\",\"arguments\":\"\",\"call_id\":\"call_7fRJCCDIZe4x17WaroXZwmjV\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"{\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"fHq8b6uDNpdKyA\",\"output_index\":0,\"sequence_number\":3}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"command\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"thvsscitV\",\"output_index\":0,\"sequence_number\":4}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"4KhNnIoFmUxL4\",\"output_index\":0,\"sequence_number\":5}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cd\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"2MAKSqM5RZc02D\",\"output_index\":0,\"sequence_number\":6}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" /\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"rtJwB0qEFcuGP0\",\"output_index\":0,\"sequence_number\":7}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ver\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"F1lTsvrOqKTGW\",\"output_index\":0,\"sequence_number\":8}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"cel\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"OW6Ho3Im2OXeD\",\"output_index\":0,\"sequence_number\":9}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/s\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"OeETspVfDui6qp\",\"output_index\":0,\"sequence_number\":10}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"andbox\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"TYKoEchhWA\",\"output_index\":0,\"sequence_number\":11}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"mlWpUOMOoFsxvPZ\",\"output_index\":0,\"sequence_number\":12}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"dhWAHpJ9eAO\",\"output_index\":0,\"sequence_number\":13}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"obL0WnjlMLS\",\"output_index\":0,\"sequence_number\":14}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"data\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"VQtHA1VRMGyD\",\"output_index\":0,\"sequence_number\":15}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"v3crkHcTsSeS6g9\",\"output_index\":0,\"sequence_number\":16}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"brain\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"fLyFqifDKnd\",\"output_index\":0,\"sequence_number\":17}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"trust\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"zOZwoEc9EwU\",\"output_index\":0,\"sequence_number\":18}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-sdk\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"jKM1tjLK0Nox\",\"output_index\":0,\"sequence_number\":19}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-j\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"N8hk8Xm3OX7w2R\",\"output_index\":0,\"sequence_number\":20}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"avascript\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"P8EgPwQ\",\"output_index\":0,\"sequence_number\":21}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"aIWWYDGHFlbnf0\",\"output_index\":0,\"sequence_number\":22}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"2\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"k9oYAIfrrIU8RoN\",\"output_index\":0,\"sequence_number\":23}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"e\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"qYFO9k1cYtpeWCN\",\"output_index\":0,\"sequence_number\":24}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/sc\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"h3KfOXIs1dENs\",\"output_index\":0,\"sequence_number\":25}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"enarios\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"JH679LrpQ\",\"output_index\":0,\"sequence_number\":26}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"/fl\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"gMCaFOZiRoUg3\",\"output_index\":0,\"sequence_number\":27}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ue\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"yXl3sFNxeXVgqy\",\"output_index\":0,\"sequence_number\":28}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"-in\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"glJkNUpBDKD1z\",\"output_index\":0,\"sequence_number\":29}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"strument\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"VBlCVyqb\",\"output_index\":0,\"sequence_number\":30}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ation\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"SNMP91Q3rDW\",\"output_index\":0,\"sequence_number\":31}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" &&\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"X3ihmCKcYuVXU\",\"output_index\":0,\"sequence_number\":32}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" node\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"JKpFNiOkxT3\",\"output_index\":0,\"sequence_number\":33}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" scenario\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"bFvkMJJ\",\"output_index\":0,\"sequence_number\":34}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".cli\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"kUiDpbheCAHH\",\"output_index\":0,\"sequence_number\":35}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\".m\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"FxsGZq8BCGo3gl\",\"output_index\":0,\"sequence_number\":36}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"js\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"Dp4QtMwr7c3BK1\",\"output_index\":0,\"sequence_number\":37}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" --\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"PeegSlydvrC1L\",\"output_index\":0,\"sequence_number\":38}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"marker\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"wvkAVwRdNi\",\"output_index\":0,\"sequence_number\":39}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\" SK\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"PTHJb9FqWdVqt\",\"output_index\":0,\"sequence_number\":40}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"ILL\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"2fQZxFddpb4N0\",\"output_index\":0,\"sequence_number\":41}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"_DONE\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"k70XHwhwure\",\"output_index\":0,\"sequence_number\":42}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\",\\\"\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"rGVgz0vPyy7dV\",\"output_index\":0,\"sequence_number\":43}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"timeout\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"MAkwSc4cj\",\"output_index\":0,\"sequence_number\":44}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"\\\":\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"RnIym4ATTjxwrH\",\"output_index\":0,\"sequence_number\":45}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"100\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"soJ0BJsFSpsnc\",\"output_index\":0,\"sequence_number\":46}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"000\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"DQKN9g0JhSsmY\",\"output_index\":0,\"sequence_number\":47}", - "event: response.function_call_arguments.delta\ndata: {\"type\":\"response.function_call_arguments.delta\",\"delta\":\"}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"obfuscation\":\"BTOAjj33gQRbswj\",\"output_index\":0,\"sequence_number\":48}", - "event: response.function_call_arguments.done\ndata: {\"type\":\"response.function_call_arguments.done\",\"arguments\":\"{\\\"command\\\":\\\"printf SKILL_DONE\\\",\\\"timeout\\\":100000}\",\"item_id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"output_index\":0,\"sequence_number\":49}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf SKILL_DONE\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_7fRJCCDIZe4x17WaroXZwmjV\",\"name\":\"bash\"},\"output_index\":0,\"sequence_number\":50}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b6f493c81a3a225e3a6b86bbee9\",\"object\":\"response\",\"created_at\":1785748335,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748336,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763\",\"type\":\"function_call\",\"status\":\"completed\",\"arguments\":\"{\\\"command\\\":\\\"printf SKILL_DONE\\\",\\\"timeout\\\":100000}\",\"call_id\":\"call_7fRJCCDIZe4x17WaroXZwmjV\",\"name\":\"bash\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":3421,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":1536},\"output_tokens\":59,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":3480},\"user\":null,\"metadata\":{}},\"sequence_number\":51}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a25433173b9c7bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:15 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "200", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179996124", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_3d947e9c4a9c4ea6a016b86fe1407743" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 14, - "id": "bc08e707f2920073", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:16.776Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run e2e-flue-skill with marker SKILL_DONE\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Ensure completion and report the final output marker only.\",\"agent\":\"e2e-flue-skill\",\"cwd\":\".\"}", - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "id": "fc_054a16a2f2ea3217016a705b64fe5481a390185c9576df12b6", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_w6pyYZgnhlSOAXul75DAG1Cj", - "output": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -name AGENTS.md -print\",\"timeout\":100000}", - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "id": "fc_054a16a2f2ea3217016a705b66460481a3a5de06df603e35ee", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_6zEBnJRmXFIkGFwNjWLZP4uC", - "output": "total 4\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 09:10 .\ndrwxr-xr-x 10 vercel-sandbox vercel-sandbox 4096 Aug 3 09:11 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 09:10 skills", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\\\;\",\"timeout\":100000}", - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "id": "fc_054a16a2f2ea3217016a705b67575881a3aacde02b9053a321", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_jQUCie0ZgaXUrZCDcezp0msl", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 6 -type f -name 'AGENTS.md' -print\",\"timeout\":100000}", - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "id": "fc_054a16a2f2ea3217016a705b68c39881a3b1e518599236e41a", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_NgMSUb6wIR0GAf9U8LXERtBQ", - "output": "(no output)", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head\",\"timeout\":100000}", - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "id": "fc_054a16a2f2ea3217016a705b69edd081a38c31583ec56eca5f", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_BXKiQ5KG8c42XPPp94zgcY3H", - "output": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print\",\"timeout\":100000}", - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "id": "fc_054a16a2f2ea3217016a705b6b96cc81a3ac4761db24198a63", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_Zmmrs64LFscpAHsIUaIfEcAN", - "output": "total 4\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 22 Aug 3 07:55 .\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 28 Aug 3 07:55 ..\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 194 Aug 3 07:55 SKILL.md\n./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "function_call_output" - }, - { - "arguments": "{\"path\":\"./.agents/skills/e2e-flue-skill/SKILL.md\",\"offset\":1,\"limit\":200}", - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "id": "fc_054a16a2f2ea3217016a705b6d71cc81a38ac06634a309a997", - "name": "read", - "type": "function_call" - }, - { - "call_id": "call_KXcpmIy4Cbfugk4SNT2Ge7sG", - "output": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"cat ./package.json\",\"timeout\":100000}", - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "id": "fc_054a16a2f2ea3217016a705b6e90f481a39bc44496fbadf441", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_xQybhnQYuflmfp7gyAScVKjY", - "output": "v24.14.1\ntotal 244\ndrwxr-xr-x 7 vercel-sandbox vercel-sandbox 4096 Aug 3 09:06 .\ndrwxr-xr-x 46 vercel-sandbox vercel-sandbox 4096 Aug 3 07:55 ..\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 20 Aug 3 07:55 .agents\ndrwxr-xr-x 3 vercel-sandbox vercel-sandbox 38 Aug 3 07:55 .flue\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 218 Aug 3 09:11 __cassettes__\ndrwxr-xr-x 2 vercel-sandbox vercel-sandbox 4096 Aug 3 08:49 __snapshots__\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 12484 Aug 3 08:45 assertions.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 380 Aug 3 07:55 cassette-filter.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 215 Aug 3 07:55 constants.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2000 Aug 3 09:06 package.json\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 167098 Aug 3 09:06 pnpm-lock.yaml\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 123 Aug 3 07:55 scenario.cli.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 6877 Aug 3 09:06 scenario.impl.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 203 Aug 3 07:55 scenario.mjs\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 8066 Aug 3 09:09 scenario.test.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 121 Aug 3 07:55 scenario.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 2376 Aug 3 08:49 scenario.v2.ts\n-rw-r--r-- 1 vercel-sandbox vercel-sandbox 4616 Aug 3 08:50 v2-assertions.ts\ndrwxr-xr-x 4 vercel-sandbox vercel-sandbox 26 Aug 3 08:06 versions\n{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.4\": \"^3.1.4\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.8\": \"^5.0.8\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\"\n }\n }\n}", - "type": "function_call_output" - }, - { - "arguments": "{\"command\":\"printf SKILL_DONE\",\"timeout\":100000}", - "call_id": "call_7fRJCCDIZe4x17WaroXZwmjV", - "id": "fc_054a16a2f2ea3217016a705b6fb64c81a38ff895c9553c8763", - "name": "bash", - "type": "function_call" - }, - { - "call_id": "call_7fRJCCDIZe4x17WaroXZwmjV", - "output": "node:internal/modules/package_json_reader:301\n throw new ERR_MODULE_NOT_FOUND(packageName, fileURLToPath(base), null);\n ^\n\nError [ERR_MODULE_NOT_FOUND]: Cannot find package '@flue/cli' imported from ./scenario.impl.mjs\n at Object.getPackageJSONURL (node:internal/modules/package_json_reader:301:9)\n at packageResolve (node:internal/modules/esm/resolve:768:81)\n at moduleResolve (node:internal/modules/esm/resolve:859:18)\n at defaultResolve (node:internal/modules/esm/resolve:991:11)\n at #cachedDefaultResolve (node:internal/modules/esm/loader:719:20)\n at #resolveAndMaybeBlockOnLoaderThread (node:internal/modules/esm/loader:736:38)\n at ModuleLoader.resolveSync (node:internal/modules/esm/loader:765:52)\n at Object.resolve (node:internal/modules/esm/initialize_import_meta:37:25)\n at file://./scenario.impl.mjs:12:33\n at ModuleJob.run (node:internal/modules/esm/module_job:430:25) {\n code: 'ERR_MODULE_NOT_FOUND'\n}\n\nNode.js v24.14.1\n\nCommand exited with code 1", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b70474881a39a050b8889ca15cd\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b70474881a39a050b8889ca15cd\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"SK\",\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"obfuscation\":\"M9SWfj095W22jm\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ILL\",\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"obfuscation\":\"NnKtLGfgvufIN\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"obfuscation\":\"wHpr2r6Jc26\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"SKILL_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_054a16a2f2ea3217016a705b70474881a39a050b8889ca15cd\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748336,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":3812,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":2560},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":3819},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a254331d7fe07bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:16 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "201", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179995734", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "1ms", - "x-request-id": "req_6dc67fd2f6124704a9d86e5e918bac37" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 15, - "id": "b3f219cac4c52d6c", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:17.515Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "developer" - }, - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "input_text" - } - ], - "role": "user" - }, - { - "arguments": "{\"description\":\"Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.\",\"prompt\":\"Run the skill named \\\"e2e-flue-skill\\\" with arguments {\\\"marker\\\":\\\"SKILL_DONE\\\"}. Output only the marker and no extra text when finished.\",\"cwd\":\".\"}", - "call_id": "call_R2a5LOml0NZqq5XTcTCKfbMW", - "id": "fc_0f099b3fce0721fd016a705b63c99081a3af4dfbf2ce0314eb", - "name": "task", - "type": "function_call" - }, - { - "call_id": "call_R2a5LOml0NZqq5XTcTCKfbMW", - "output": "SKILL_DONE", - "type": "function_call_output" - } - ], - "model": "gpt-5.4-nano", - "prompt_cache_key": "[REDACTED]", - "reasoning": { - "effort": "none" - }, - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b70d65881a3b5bc873ea54e7c00\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b70d65881a3b5bc873ea54e7c00\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"SK\",\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"obfuscation\":\"iSZ7rWanncJQ68\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ILL\",\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"obfuscation\":\"PQly4lHaqRXSu\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"obfuscation\":\"WSIPPwqqidY\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":7,\"text\":\"SKILL_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"},\"sequence_number\":8}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":9}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f099b3fce0721fd016a705b70d65881a3b5bc873ea54e7c00\",\"object\":\"response\",\"created_at\":1785748336,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748337,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-5.4-nano-2026-03-17\",\"moderation\":null,\"output\":[{\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"SKILL_DONE\"}],\"phase\":\"final_answer\",\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::sk\",\"prompt_cache_retention\":\"24h\",\"reasoning\":{\"context\":\"current_turn\",\"effort\":\"none\",\"mode\":\"standard\",\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":0.98,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":969,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":7,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":976},\"user\":null,\"metadata\":{}},\"sequence_number\":10}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a2543320eec07bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:17 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "226", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-ratelimit-limit-requests": "30000", - "x-ratelimit-limit-tokens": "180000000", - "x-ratelimit-remaining-requests": "29999", - "x-ratelimit-remaining-tokens": "179998575", - "x-ratelimit-reset-requests": "2ms", - "x-ratelimit-reset-tokens": "0s", - "x-request-id": "req_0f63631954824ce89d221047a6871115" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 16, - "id": "eea2c2f028018d92", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:18.781Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are running in headless mode with no human operator. Work autonomously — never ask questions, never wait for user input. Make your best judgment and proceed independently.\n\nYou are a deterministic Flue instrumentation test agent. Follow user instructions exactly. When asked for a marker, output only that marker and no extra text. When running a local skill file, read it yourself and do not delegate it to a task.\n\n## Available Skills\n\nThe following skills provide specialized instructions for specific tasks. When a task matches a skill description, activate that skill before proceeding so its full instructions are loaded. Skill instructions and supporting resources stay lazy until activation or explicit file reads.\n\n- **e2e-flue-skill** — Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n\nDate: Mon, Aug 3, 2026\nWorking directory: .\n\nDirectory structure:\n.agents\n.flue\n.flue-build\n__cassettes__\n__snapshots__\nassertions.ts\ncassette-filter.mjs\nconstants.mjs\ndist\nnode_modules\npackage.json\npnpm-lock.yaml\nscenario.cli.mjs\nscenario.impl.mjs\nscenario.mjs\nscenario.test.ts\nscenario.ts\nscenario.v2.ts\nv2-assertions.ts\nversions", - "role": "system" - }, - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "model": "gpt-4o-mini", - "prompt_cache_key": "[REDACTED]", - "store": false, - "stream": true, - "tools": [ - { - "description": "Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.", - "name": "read", - "parameters": { - "properties": { - "limit": { - "description": "Maximum number of lines to read", - "type": "number" - }, - "offset": { - "description": "Line number to start from (1-indexed)", - "type": "number" - }, - "path": { - "description": "Path to the file to read", - "type": "string" - } - }, - "required": ["path"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Write content to a file. Creates the file and parent directories if they do not exist.", - "name": "write", - "parameters": { - "properties": { - "content": { - "description": "Content to write to the file", - "type": "string" - }, - "path": { - "description": "Path to the file to write", - "type": "string" - } - }, - "required": ["path", "content"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.", - "name": "edit", - "parameters": { - "properties": { - "newText": { - "description": "Replacement text", - "type": "string" - }, - "oldText": { - "description": "Exact text to find (must be unique)", - "type": "string" - }, - "path": { - "description": "Path to the file to edit", - "type": "string" - }, - "replaceAll": { - "description": "Replace all occurrences", - "type": "boolean" - } - }, - "required": ["path", "oldText", "newText"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.", - "name": "bash", - "parameters": { - "properties": { - "command": { - "description": "Bash command to execute", - "type": "string" - }, - "timeout": { - "description": "Timeout in seconds", - "type": "number" - } - }, - "required": ["command"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.", - "name": "grep", - "parameters": { - "properties": { - "include": { - "description": "Glob filter, e.g. \"*.ts\"", - "type": "string" - }, - "path": { - "description": "Directory or file to search (default: .)", - "type": "string" - }, - "pattern": { - "description": "Search pattern (regex)", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Find files by filename pattern using shell find -name semantics. Returns matching file paths.", - "name": "glob", - "parameters": { - "properties": { - "path": { - "description": "Directory to search in (default: .)", - "type": "string" - }, - "pattern": { - "description": "Filename pattern, e.g. \"*.ts\"", - "type": "string" - } - }, - "required": ["pattern"], - "type": "object" - }, - "strict": false, - "type": "function" - }, - { - "description": "Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.", - "name": "task", - "parameters": { - "properties": { - "agent": { - "description": "Declared subagent to use for the child agent", - "type": "string" - }, - "cwd": { - "description": "Working directory for the child agent. AGENTS.md and skills are discovered from here.", - "type": "string" - }, - "description": { - "description": "Short human-readable label for the delegated work", - "type": "string" - }, - "prompt": { - "description": "Focused instructions for the child agent", - "type": "string" - } - }, - "required": ["prompt"], - "type": "object" - }, - "strict": false, - "type": "function" - } - ] - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_0f3adfdb1510c165016a705b7192bc81a094e1dbada8d8b42b\",\"object\":\"response\",\"created_at\":1785748337,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_0f3adfdb1510c165016a705b7192bc81a094e1dbada8d8b42b\",\"object\":\"response\",\"created_at\":1785748337,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"TASK\",\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"logprobs\":[],\"obfuscation\":\"mlb5p2sBa8Nm\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"_DONE\",\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"logprobs\":[],\"obfuscation\":\"oWnwIAUMpIA\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":6,\"text\":\"TASK_DONE\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"},\"sequence_number\":7}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":8}", - "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_0f3adfdb1510c165016a705b7192bc81a094e1dbada8d8b42b\",\"object\":\"response\",\"created_at\":1785748337,\"status\":\"completed\",\"background\":false,\"completed_at\":1785748338,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"TASK_DONE\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":\"workflow:instrumentation:01KZ3E9TWTAEK4P4P3F63HKAKC::default::ta\",\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[{\"type\":\"function\",\"description\":\"Read a file or list a directory. For files, output is truncated to 2000 lines or 50KB — use offset/limit for large files. For directories, returns the list of entries.\",\"name\":\"read\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to read\"},\"offset\":{\"type\":\"number\",\"description\":\"Line number to start from (1-indexed)\"},\"limit\":{\"type\":\"number\",\"description\":\"Maximum number of lines to read\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Write content to a file. Creates the file and parent directories if they do not exist.\",\"name\":\"write\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"content\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to write\"},\"content\":{\"type\":\"string\",\"description\":\"Content to write to the file\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Edit a file using exact text replacement. The oldText must match a unique region of the file. Use replaceAll to replace all occurrences.\",\"name\":\"edit\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"path\",\"oldText\",\"newText\"],\"properties\":{\"path\":{\"type\":\"string\",\"description\":\"Path to the file to edit\"},\"oldText\":{\"type\":\"string\",\"description\":\"Exact text to find (must be unique)\"},\"newText\":{\"type\":\"string\",\"description\":\"Replacement text\"},\"replaceAll\":{\"type\":\"boolean\",\"description\":\"Replace all occurrences\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Execute a bash command. Returns stdout and stderr. Output is truncated to the last 2000 lines or 50KB.\",\"name\":\"bash\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"command\"],\"properties\":{\"command\":{\"type\":\"string\",\"description\":\"Bash command to execute\"},\"timeout\":{\"type\":\"number\",\"description\":\"Timeout in seconds\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Search file contents for a regex pattern. Returns matching lines with file paths and line numbers.\",\"name\":\"grep\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Search pattern (regex)\"},\"path\":{\"type\":\"string\",\"description\":\"Directory or file to search (default: .)\"},\"include\":{\"type\":\"string\",\"description\":\"Glob filter, e.g. \\\"*.ts\\\"\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Find files by filename pattern using shell find -name semantics. Returns matching file paths.\",\"name\":\"glob\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"pattern\"],\"properties\":{\"pattern\":{\"type\":\"string\",\"description\":\"Filename pattern, e.g. \\\"*.ts\\\"\"},\"path\":{\"type\":\"string\",\"description\":\"Directory to search in (default: .)\"}}},\"strict\":false},{\"type\":\"function\",\"description\":\"Delegate a focused task to a detached child agent with its own context. Use this for independent research, file exploration, or parallel work. The task returns only its final answer to this conversation. No subagents are currently defined.\",\"name\":\"task\",\"output_schema\":null,\"parameters\":{\"type\":\"object\",\"required\":[\"prompt\"],\"properties\":{\"description\":{\"type\":\"string\",\"description\":\"Short human-readable label for the delegated work\"},\"prompt\":{\"type\":\"string\",\"description\":\"Focused instructions for the child agent\"},\"agent\":{\"type\":\"string\",\"description\":\"Declared subagent to use for the child agent\"},\"cwd\":{\"type\":\"string\",\"description\":\"Working directory for the child agent. AGENTS.md and skills are discovered from here.\"}}},\"strict\":false}],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":826,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":4,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":830},\"user\":null,\"metadata\":{}},\"sequence_number\":9}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a254332588917bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:17 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "169", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_44ff6ef3357a4d78a0e4b836a0777717" - }, - "status": 200, - "statusText": "OK" - } - }, - { - "callIndex": 17, - "id": "dd8319160ba3e4cf", - "matchKey": "POST api.openai.com/v1/responses", - "recordedAt": "2026-08-03T09:12:19.721Z", - "request": { - "body": { - "kind": "json", - "value": { - "input": [ - { - "content": "You are a context summarization assistant. Your task is to read a conversation between a user and an AI coding assistant, then produce a structured summary following the exact format specified.\n\nDo NOT continue the conversation. Do NOT respond to any questions in the conversation. ONLY output the structured summary.", - "role": "system" - }, - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "input_text" - } - ], - "role": "user" - } - ], - "max_output_tokens": 32, - "model": "gpt-4o-mini", - "store": false, - "stream": true - } - }, - "headers": {}, - "method": "POST", - "url": "https://api.openai.com/v1/responses" - }, - "response": { - "body": { - "chunks": [ - "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_04d28ba03840bb65016a705b72d36081a3a387e30f2739466c\",\"object\":\"response\",\"created_at\":1785748338,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}", - "event: response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_04d28ba03840bb65016a705b72d36081a3a387e30f2739466c\",\"object\":\"response\",\"created_at\":1785748338,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}", - "event: response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":2}", - "event: response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":3}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"##\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"pnRDruuzK4zmSi\",\"output_index\":0,\"sequence_number\":4}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Original\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"xeQeZBG\",\"output_index\":0,\"sequence_number\":5}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" Request\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"wDl6Xfa7\",\"output_index\":0,\"sequence_number\":6}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"SPfPR8TAgZd37wg\",\"output_index\":0,\"sequence_number\":7}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"The\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"cgHwyTaOZmFtZ\",\"output_index\":0,\"sequence_number\":8}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" user\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"1z0yUfwbdDc\",\"output_index\":0,\"sequence_number\":9}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" requested\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"eCqgHs\",\"output_index\":0,\"sequence_number\":10}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" a\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"7RAJkk4N6FYOim\",\"output_index\":0,\"sequence_number\":11}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" step\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"y7scL3Od23b\",\"output_index\":0,\"sequence_number\":12}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-by\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"W7lrv8VHEOIfi\",\"output_index\":0,\"sequence_number\":13}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"-step\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"4LIgLNAJAB0\",\"output_index\":0,\"sequence_number\":14}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" execution\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"F077yL\",\"output_index\":0,\"sequence_number\":15}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" of\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"fMOsa5wMUpP8t\",\"output_index\":0,\"sequence_number\":16}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" an\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"SXWFdQci5VvIi\",\"output_index\":0,\"sequence_number\":17}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" instrument\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"IM33q\",\"output_index\":0,\"sequence_number\":18}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ed\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"nO8YVpkRDW8oOQ\",\"output_index\":0,\"sequence_number\":19}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" research\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"xjbeOFx\",\"output_index\":0,\"sequence_number\":20}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" flow\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"PjjjDiy3SBJ\",\"output_index\":0,\"sequence_number\":21}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" involving\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"I58VnM\",\"output_index\":0,\"sequence_number\":22}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" tool\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"i0bH5Hefbkz\",\"output_index\":0,\"sequence_number\":23}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" calls\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"BEMMaOeJJU\",\"output_index\":0,\"sequence_number\":24}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" to\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"gPeC3aszPMi0S\",\"output_index\":0,\"sequence_number\":25}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" gather\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"zlkkxcR5g\",\"output_index\":0,\"sequence_number\":26}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" information\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"7hhs\",\"output_index\":0,\"sequence_number\":27}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" on\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"lLr5hZIXvG8Gb\",\"output_index\":0,\"sequence_number\":28}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \\\"\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"5qdl5MlY2bxs7F\",\"output_index\":0,\"sequence_number\":29}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"fl\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"yt6z8GDsVV0Qew\",\"output_index\":0,\"sequence_number\":30}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"ue\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"3QmzDfnfod8ZRQ\",\"output_index\":0,\"sequence_number\":31}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" instrumentation\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"\",\"output_index\":0,\"sequence_number\":32}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\\"\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"yiSjj02o3FbcC4U\",\"output_index\":0,\"sequence_number\":33}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" and\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"796Vjv2p8Npd\",\"output_index\":0,\"sequence_number\":34}", - "event: response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" \\\"\",\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"obfuscation\":\"WkdXPw3c1tG4kc\",\"output_index\":0,\"sequence_number\":35}", - "event: response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"logprobs\":[],\"output_index\":0,\"sequence_number\":36,\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"}", - "event: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"output_index\":0,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"},\"sequence_number\":37}", - "event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"}],\"role\":\"assistant\"},\"output_index\":0,\"sequence_number\":38}", - "event: response.incomplete\ndata: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_04d28ba03840bb65016a705b72d36081a3a387e30f2739466c\",\"object\":\"response\",\"created_at\":1785748338,\"status\":\"incomplete\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0,\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"instructions\":null,\"max_output_tokens\":32,\"max_tool_calls\":null,\"model\":\"gpt-4o-mini-2024-07-18\",\"moderation\":null,\"output\":[{\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\",\"type\":\"message\",\"status\":\"incomplete\",\"content\":[{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"## Original Request\\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \\\"flue instrumentation\\\" and \\\"\"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":\"in_memory\",\"reasoning\":{\"context\":null,\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"default\",\"store\":false,\"temperature\":1,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":\"auto\",\"tool_usage\":{\"image_gen\":{\"input_tokens\":0,\"input_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"image_tokens\":0,\"text_tokens\":0},\"total_tokens\":0},\"web_search\":{\"num_requests\":0}},\"tools\":[],\"top_logprobs\":0,\"top_p\":1,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":446,\"input_tokens_details\":{\"cache_write_tokens\":0,\"cached_tokens\":0},\"output_tokens\":32,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":478},\"user\":null,\"metadata\":{}},\"sequence_number\":39}" - ], - "kind": "sse" - }, - "headers": { - "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", - "alt-svc": "h3=\":443\"; ma=86400", - "cf-cache-status": "DYNAMIC", - "cf-ray": "a254332d68c17bb6-IAD", - "connection": "keep-alive", - "content-type": "text/event-stream; charset=utf-8", - "date": "Mon, 03 Aug 2026 09:12:18 GMT", - "openai-organization": "braintrust-data", - "openai-processing-ms": "159", - "openai-project": "proj_vsCSXafhhByzWOThMrJcZiw9", - "openai-version": "2020-10-01", - "server": "cloudflare", - "set-cookie": "[REDACTED]", - "strict-transport-security": "max-age=31536000; includeSubDomains; preload", - "transfer-encoding": "chunked", - "x-content-type-options": "nosniff", - "x-request-id": "req_7ea02cb4cc7c43e08845985aea9af023" - }, - "status": 200, - "statusText": "OK" - } - } - ], - "meta": { - "createdAt": "2026-08-03T09:07:11.649Z" - } -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json deleted file mode 100644 index dd92abe29..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.json +++ /dev/null @@ -1,1448 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 89, - "duration_ms": 0, - "estimated_cost": 0.0041862, - "prompt_cache_creation_tokens": 670, - "prompt_cached_tokens": 1099, - "prompt_tokens": 3, - "tokens": 1861 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "output": { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 52, - "duration_ms": 0, - "estimated_cost": 0.00201795, - "prompt_cache_creation_tokens": 183, - "prompt_cached_tokens": 1769, - "prompt_tokens": 7, - "tokens": 2011 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents" - }, - "output": { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 54, - "duration_ms": 0, - "estimated_cost": 0.0016611, - "prompt_cache_creation_tokens": 66, - "prompt_cached_tokens": 1952, - "prompt_tokens": 6, - "tokens": 2078 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills" - }, - "output": { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 63, - "duration_ms": 0, - "estimated_cost": 0.0018459, - "prompt_cache_creation_tokens": 74, - "prompt_cached_tokens": 2018, - "prompt_tokens": 6, - "tokens": 2161 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "output": { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills/e2e-flue-skill" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.0019618500000000002, - "prompt_cache_creation_tokens": 79, - "prompt_cached_tokens": 2092, - "prompt_tokens": 6, - "tokens": 2245 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 8, - "duration_ms": 0, - "estimated_cost": 0.00129555, - "prompt_cache_creation_tokens": 135, - "prompt_cached_tokens": 2171, - "prompt_tokens": 6, - "tokens": 2320 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.00012299999999999998, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 804, - "tokens": 808 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", - "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_mode": "full", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 196, - "duration_ms": 0, - "estimated_cost": 0.0113025, - "prompt_cache_creation_tokens": 2222, - "prompt_cached_tokens": 0, - "prompt_tokens": 10, - "tokens": 2428 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 158, - "duration_ms": 0, - "estimated_cost": 0.0039606, - "prompt_cache_creation_tokens": 236, - "prompt_cached_tokens": 2222, - "prompt_tokens": 13, - "tokens": 2629 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 138, - "duration_ms": 0, - "estimated_cost": 0.0037239000000000005, - "prompt_cache_creation_tokens": 234, - "prompt_cached_tokens": 2458, - "prompt_tokens": 13, - "tokens": 2843 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt deleted file mode 100644 index e90a4892a..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-auto-hook.span-tree.txt +++ /dev/null @@ -1,1335 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 89, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0041862, -│ │ │ "prompt_cache_creation_tokens": 670, -│ │ │ "prompt_cached_tokens": 1099, -│ │ │ "prompt_tokens": 3, -│ │ │ "tokens": 1861 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": {} -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 52, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00201795, -│ │ │ "prompt_cache_creation_tokens": 183, -│ │ │ "prompt_cached_tokens": 1769, -│ │ │ "prompt_tokens": 7, -│ │ │ "tokens": 2011 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 54, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0016611, -│ │ │ "prompt_cache_creation_tokens": 66, -│ │ │ "prompt_cached_tokens": 1952, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2078 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 63, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0018459, -│ │ │ "prompt_cache_creation_tokens": 74, -│ │ │ "prompt_cached_tokens": 2018, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2161 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 68, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0019618500000000002, -│ │ │ "prompt_cache_creation_tokens": 79, -│ │ │ "prompt_cached_tokens": 2092, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2245 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "lines": 7, -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ "type": "text" -│ │ }, -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/e2e-flue-skill.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": true, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "skills", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "e2e-flue-skill", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL.md", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "anthropic-messages", -│ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ "flue.provider": "anthropic", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "claude-sonnet-4-5-20250929", -│ │ "provider": "anthropic" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 8, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00129555, -│ │ "prompt_cache_creation_tokens": 135, -│ │ "prompt_cached_tokens": 2171, -│ │ "prompt_tokens": 6, -│ │ "tokens": 2320 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00012299999999999998, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 804, -│ │ "tokens": 808 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", -│ "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_mode": "full", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "duration_ms": 0, - │ "estimated_cost": 0.0113025, - │ "prompt_cache_creation_tokens": 2222, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 10, - │ "tokens": 2428 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 158, - │ "duration_ms": 0, - │ "estimated_cost": 0.0039606, - │ "prompt_cache_creation_tokens": 236, - │ "prompt_cached_tokens": 2222, - │ "prompt_tokens": 13, - │ "tokens": 2629 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - │ "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 3:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 138, - │ "duration_ms": 0, - │ "estimated_cost": 0.0037239000000000005, - │ "prompt_cache_creation_tokens": 234, - │ "prompt_cached_tokens": 2458, - │ "prompt_tokens": 13, - │ "tokens": 2843 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json deleted file mode 100644 index dd92abe29..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.json +++ /dev/null @@ -1,1448 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 89, - "duration_ms": 0, - "estimated_cost": 0.0041862, - "prompt_cache_creation_tokens": 670, - "prompt_cached_tokens": 1099, - "prompt_tokens": 3, - "tokens": 1861 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "output": { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 52, - "duration_ms": 0, - "estimated_cost": 0.00201795, - "prompt_cache_creation_tokens": 183, - "prompt_cached_tokens": 1769, - "prompt_tokens": 7, - "tokens": 2011 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents" - }, - "output": { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 54, - "duration_ms": 0, - "estimated_cost": 0.0016611, - "prompt_cache_creation_tokens": 66, - "prompt_cached_tokens": 1952, - "prompt_tokens": 6, - "tokens": 2078 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills" - }, - "output": { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 63, - "duration_ms": 0, - "estimated_cost": 0.0018459, - "prompt_cache_creation_tokens": 74, - "prompt_cached_tokens": 2018, - "prompt_tokens": 6, - "tokens": 2161 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "output": { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills/e2e-flue-skill" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.0019618500000000002, - "prompt_cache_creation_tokens": 79, - "prompt_cached_tokens": 2092, - "prompt_tokens": 6, - "tokens": 2245 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 8, - "duration_ms": 0, - "estimated_cost": 0.00129555, - "prompt_cache_creation_tokens": 135, - "prompt_cached_tokens": 2171, - "prompt_tokens": 6, - "tokens": 2320 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.00012299999999999998, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 804, - "tokens": 808 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", - "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_mode": "full", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 196, - "duration_ms": 0, - "estimated_cost": 0.0113025, - "prompt_cache_creation_tokens": 2222, - "prompt_cached_tokens": 0, - "prompt_tokens": 10, - "tokens": 2428 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 158, - "duration_ms": 0, - "estimated_cost": 0.0039606, - "prompt_cache_creation_tokens": 236, - "prompt_cached_tokens": 2222, - "prompt_tokens": 13, - "tokens": 2629 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 138, - "duration_ms": 0, - "estimated_cost": 0.0037239000000000005, - "prompt_cache_creation_tokens": 234, - "prompt_cached_tokens": 2458, - "prompt_tokens": 13, - "tokens": 2843 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt deleted file mode 100644 index e90a4892a..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-cli.span-tree.txt +++ /dev/null @@ -1,1335 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 89, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0041862, -│ │ │ "prompt_cache_creation_tokens": 670, -│ │ │ "prompt_cached_tokens": 1099, -│ │ │ "prompt_tokens": 3, -│ │ │ "tokens": 1861 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": {} -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 52, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00201795, -│ │ │ "prompt_cache_creation_tokens": 183, -│ │ │ "prompt_cached_tokens": 1769, -│ │ │ "prompt_tokens": 7, -│ │ │ "tokens": 2011 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 54, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0016611, -│ │ │ "prompt_cache_creation_tokens": 66, -│ │ │ "prompt_cached_tokens": 1952, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2078 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 63, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0018459, -│ │ │ "prompt_cache_creation_tokens": 74, -│ │ │ "prompt_cached_tokens": 2018, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2161 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 68, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0019618500000000002, -│ │ │ "prompt_cache_creation_tokens": 79, -│ │ │ "prompt_cached_tokens": 2092, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2245 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "lines": 7, -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ "type": "text" -│ │ }, -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/e2e-flue-skill.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": true, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "skills", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "e2e-flue-skill", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL.md", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "anthropic-messages", -│ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ "flue.provider": "anthropic", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "claude-sonnet-4-5-20250929", -│ │ "provider": "anthropic" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 8, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00129555, -│ │ "prompt_cache_creation_tokens": 135, -│ │ "prompt_cached_tokens": 2171, -│ │ "prompt_tokens": 6, -│ │ "tokens": 2320 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00012299999999999998, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 804, -│ │ "tokens": 808 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", -│ "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_mode": "full", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "duration_ms": 0, - │ "estimated_cost": 0.0113025, - │ "prompt_cache_creation_tokens": 2222, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 10, - │ "tokens": 2428 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 158, - │ "duration_ms": 0, - │ "estimated_cost": 0.0039606, - │ "prompt_cache_creation_tokens": 236, - │ "prompt_cached_tokens": 2222, - │ "prompt_tokens": 13, - │ "tokens": 2629 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - │ "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 3:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 138, - │ "duration_ms": 0, - │ "estimated_cost": 0.0037239000000000005, - │ "prompt_cache_creation_tokens": 234, - │ "prompt_cached_tokens": 2458, - │ "prompt_tokens": 13, - │ "tokens": 2843 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json deleted file mode 100644 index dd92abe29..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.json +++ /dev/null @@ -1,1448 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 89, - "duration_ms": 0, - "estimated_cost": 0.0041862, - "prompt_cache_creation_tokens": 670, - "prompt_cached_tokens": 1099, - "prompt_tokens": 3, - "tokens": 1861 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/e2e-flue-skill.md" - }, - "output": { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 52, - "duration_ms": 0, - "estimated_cost": 0.00201795, - "prompt_cache_creation_tokens": 183, - "prompt_cached_tokens": 1769, - "prompt_tokens": 7, - "tokens": 2011 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents" - }, - "output": { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 54, - "duration_ms": 0, - "estimated_cost": 0.0016611, - "prompt_cache_creation_tokens": 66, - "prompt_cached_tokens": 1952, - "prompt_tokens": 6, - "tokens": 2078 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills" - }, - "output": { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 63, - "duration_ms": 0, - "estimated_cost": 0.0018459, - "prompt_cache_creation_tokens": 74, - "prompt_cached_tokens": 2018, - "prompt_tokens": 6, - "tokens": 2161 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill" - }, - "output": { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "details": { - "entries": 1, - "isDirectory": true, - "path": ".agents/skills/e2e-flue-skill" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.0019618500000000002, - "prompt_cache_creation_tokens": 79, - "prompt_cached_tokens": 2092, - "prompt_tokens": 6, - "tokens": 2245 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", - "type": "text" - }, - { - "arguments": { - "path": ".agents/e2e-flue-skill.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "skills", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - }, - { - "content": [ - { - "arguments": { - "path": ".agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 8, - "duration_ms": 0, - "estimated_cost": 0.00129555, - "prompt_cache_creation_tokens": 135, - "prompt_cached_tokens": 2171, - "prompt_tokens": 6, - "tokens": 2320 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.00012299999999999998, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 804, - "tokens": 808 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", - "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 2950, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_mode": "full", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 196, - "duration_ms": 0, - "estimated_cost": 0.0113025, - "prompt_cache_creation_tokens": 2222, - "prompt_cached_tokens": 0, - "prompt_tokens": 10, - "tokens": 2428 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - "type": "thinking" - }, - { - "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - "type": "text" - }, - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 158, - "duration_ms": 0, - "estimated_cost": 0.0039606, - "prompt_cache_creation_tokens": 236, - "prompt_cached_tokens": 2222, - "prompt_tokens": 13, - "tokens": 2629 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 2:", - "type": "text" - }, - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 138, - "duration_ms": 0, - "estimated_cost": 0.0037239000000000005, - "prompt_cache_creation_tokens": 234, - "prompt_cached_tokens": 2458, - "prompt_tokens": 13, - "tokens": 2843 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - }, - "metrics": { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt deleted file mode 100644 index e90a4892a..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-0-explicit.span-tree.txt +++ /dev/null @@ -1,1335 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 89, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0041862, -│ │ │ "prompt_cache_creation_tokens": 670, -│ │ │ "prompt_cached_tokens": 1099, -│ │ │ "prompt_tokens": 3, -│ │ │ "tokens": 1861 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": {} -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'\"}],\"details\":{}}" -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 52, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00201795, -│ │ │ "prompt_cache_creation_tokens": 183, -│ │ │ "prompt_cached_tokens": 1769, -│ │ │ "prompt_tokens": 7, -│ │ │ "tokens": 2011 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 54, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0016611, -│ │ │ "prompt_cache_creation_tokens": 66, -│ │ │ "prompt_cached_tokens": 1952, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2078 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 63, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0018459, -│ │ │ "prompt_cache_creation_tokens": 74, -│ │ │ "prompt_cached_tokens": 2018, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2161 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "entries": 1, -│ │ │ "isDirectory": true, -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ │ "type": "text" -│ │ │ }, -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/e2e-flue-skill.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": true, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "skills", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "e2e-flue-skill", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL.md", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "read" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "read", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "anthropic-messages", -│ │ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ │ "flue.provider": "anthropic", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "claude-sonnet-4-5-20250929", -│ │ │ "provider": "anthropic" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 68, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.0019618500000000002, -│ │ │ "prompt_cache_creation_tokens": 79, -│ │ │ "prompt_cached_tokens": 2092, -│ │ │ "prompt_tokens": 6, -│ │ │ "tokens": 2245 -│ │ │ } -│ │ ├── tool:read [tool] -│ │ │ input: { -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "lines": 7, -│ │ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "read", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "I'll run the e2e-flue-skill. Let me first read the skill file to understand its instructions.", -│ │ "type": "text" -│ │ }, -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/e2e-flue-skill.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "ENOENT: no such file or directory, open '/e2e/scenarios/flue-instrumentation/.agents/e2e-flue-skill.md'", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": true, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "skills", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "e2e-flue-skill", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL.md", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "path": ".agents/skills/e2e-flue-skill/SKILL.md" -│ │ }, -│ │ "id": "", -│ │ "name": "read", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "read" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "anthropic-messages", -│ │ "flue.model": "claude-sonnet-4-5-20250929", -│ │ "flue.provider": "anthropic", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "claude-sonnet-4-5-20250929", -│ │ "provider": "anthropic" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 8, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00129555, -│ │ "prompt_cache_creation_tokens": 135, -│ │ "prompt_cached_tokens": 2171, -│ │ "prompt_tokens": 6, -│ │ "tokens": 2320 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_09b32b2821e9a4d8016a184600019081929d34bb7d4c6a246f\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00012299999999999998, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 804, -│ │ "tokens": 808 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 2950, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant thinking]: The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.\n\n[Assistant]: I'll complete this instrumented research flow step by step. Starting with Step 1:\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant thinking]: Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".\n\n[Assistant]: Now proceeding to Step 2:\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant thinking]: Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.\n\n[Assistant]: Now proceeding to Step 3:\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested to complete an instrumented research flow consisting of three specific steps, involving calling various tools in a sequential manner.\n\n## Early Progress", -│ "textSignature": "{\"v\":1,\"id\":\"msg_01ecf68325f9e202016a184601309081a0878ef4054e950d32\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_mode": "full", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 196, - │ "duration_ms": 0, - │ "estimated_cost": 0.0113025, - │ "prompt_cache_creation_tokens": 2222, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 10, - │ "tokens": 2428 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "The user wants me to complete a research flow with exactly three steps:\n1. Call lookup with query \"flue instrumentation\"\n2. Call web_search with the lookupId from step 1 and query \"Braintrust Flue reasoning stream instrumentation\"\n3. Call summarize_source with the first URL from step 2\n4. Reply with exactly \"PROMPT_DONE\"\n\nThe user emphasizes I should call exactly one tool per turn and wait for each result. Let me start with step 1.", - │ "thinkingSignature": "EoQFCm4IDhgCKkD+AoKMhpXNPUJ5eANwM28uUCJPCE+AXh0ACw7aXniX+YN04zdJl3hlHMbtAuKhhczLiEeOJtu4KtVpea5u5NFdMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMIOTWCNEtsiI9vF4SGgwPXC861uZx8lpOziEiMLsCukGIluD0U44degzYnHFc+BwXNIEB+yatSHlZE+a5smr9crZcft3OLmsIdXuaFSrDAyQxWd9oqfJdnvza9eWIaOCeNkVCjrTc/wotBwh5Xp8m9k/WKt7P4zzUJBuLQ1f6ZnsdWkL8OiL/2E7Up55z2+UuVWsg1Y90/x27YRFdQjv3RnztYp4bZ5oIHAlTJoCzWiBYEh/Z8c2O+wy6MHo1cp0T57OroB/4HAqRZxrjTOXpKYLmYKXAY5tU8DTd3A0/FfEhg3oPSbOhX+112szFCpm3BRNJYooUUhwSu1hO4H/6r7d/Yp9XIM4tBfzuDx83CC/YetdOFgnYkdliw0xaUGaj4o37rWIRctWHAMdlUc4NDA0Ljc1uirHDXkN8EJwWSoIjPSBTA/p/wgLrvRV5Yy9Z0dMoWOpgYF6t1H/tNAyY8ml1CA7IWyd3jnfTZ1/G3Y0xFDwJDFlb4/QKc67w36SSCa+X9mMoTxD/DYN249lsII+XX3lQ7F1zJ07THOtCyRirCaRhmE/kGwl5tiB1QHrWa9bNHxQxf9i92435JGKN053Fe14AR70MtBSzIU2/zbgvhB1vF5lZt2i46hNxfo4lXR495gEwdwC6+8HJnlEJAJkdeHfZKj+RiwXoiYKPKsSbHi83X8Hh32IifuglNxdlxVQYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "I'll complete this instrumented research flow step by step. Starting with Step 1:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 158, - │ "duration_ms": 0, - │ "estimated_cost": 0.0039606, - │ "prompt_cache_creation_tokens": 236, - │ "prompt_cached_tokens": 2222, - │ "prompt_tokens": 13, - │ "tokens": 2629 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "thinking": "Good, I got the lookup result with id \"flue-session-2026\". Now I need to proceed to step 2: call web_search with this lookupId and the query \"Braintrust Flue reasoning stream instrumentation\".", - │ "thinkingSignature": "EpEDCm4IDhgCKkAMahve1kB1j4njp1ZSzO8eCK4bb5Ex7MB6tjdoOXfuUDRfkBqvIGdwVly8GiA8GGYC7Y/RVfREXF6A6OXHTwFyMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMLw+fsa4yo7a7ZSoIGgyaTdvH020n2dQj9bUiMB4OokY29w+uZb4rqDnKDbLXUqUj6U0f1QCGQF2ZBYDDfDJyM5Jv1xY3rQiUUjkhBirQAS4XivjvMnWI5zHBmpI/u/pq4aGdrdb3vUsJ/ne6R1d80JA/INGF8vLkQcZ5xxEK0jU4yRE+ldc2uzavmtoxIKAFD6bqKks1Qv6u+TIZtuW7tUB/7jFxDw93KmJNruMIa8c1J6pPUyBy3xE7QwgmU0kDfIfdnljxLoEBiMwtnxjCPj9lT7HHz4FkrhbnQTV4MvAQohCLjaSbOTd6LdLxwtJPm9mc26ID4eYN1gCCzyAxGXhqLHlzc23PafvQjsZliWnLhaiNMnG0YEo78XkuJQsYAQ==", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 2:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - │ "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - │ "type": "thinking" - │ }, - │ { - │ "text": "Now proceeding to Step 3:", - │ "type": "text" - │ }, - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "anthropic-messages", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "claude-sonnet-4-5-20250929", - │ "flue.provider": "anthropic", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "claude-sonnet-4-5-20250929", - │ "provider": "anthropic" - │ } - │ metrics: { - │ "completion_tokens": 138, - │ "duration_ms": 0, - │ "estimated_cost": 0.0037239000000000005, - │ "prompt_cache_creation_tokens": 234, - │ "prompt_cached_tokens": 2458, - │ "prompt_tokens": 13, - │ "tokens": 2843 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "thinking": "Perfect! I got the web_search results with one result. The first (and only) result has the URL \"https://example.test/flue/reasoning-streams\". Now I need to proceed to step 3: call summarize_source with this URL.", - "thinkingSignature": "EqQDCm4IDhgCKkAXv7EDAhWMcrBz11bUd5qxlzyy03CAGQfRgLHDE0uKzLATh1o2sedd8GxLwwQqV0CmLRfNxsdK/ZDp3Lks9iysMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIMXDMniOy5Dy0KI8x7GgzgHs7BYJMs6/N0ur8iMPe0SwnHhP39OjEu7P8ASwK9ffwa3VP67QOlPzOstRRdEZNd0mq+BVjpmBXB+QcAzCrjAeAh/UO/d7SfFmXjqwA3//thoL/TTlGa6W+fdOCFRikn65JWT4ZdVFetfqD+9/QCP/nXEwMsLnTBAxC4EGryrgs2YT+OWwmEJ7rHKFeDs5djhwSb2J2ZJO1kRvD9s2vCCDkImygYBSIVgx1Y0CZJHhzn2Vf39v0Fbth2wDZa+OqxivV9NmDZEjmgcM+tTRx1CCnaww8LZEJQ68v/dUJy7hhD4MrNcQ62evJYPfT3+4EKKWJ/7HI7KXz91//+YAkph63XstbJMFJ3AldAFiq3ecauXt5r42N5gmCmwZX+sI48ENLGGAE=", - "type": "thinking" - }, - { - "text": "Now proceeding to Step 3:", - "type": "text" - }, - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "thinking": "Perfect! The summarize_source call has completed successfully. According to the instructions, I should now reply with exactly \"PROMPT_DONE\" and no other text.", - "thinkingSignature": "Eu8CCm4IDhgCKkAn2QOIIlYfvcs6ynsmhCslHDtBsrTA4CfmimK7zyz6+k6dZldV2JBtEBET41It0gFcGEKC0n0mAhaYVuLutpDDMhpjbGF1ZGUtc29ubmV0LTQtNS0yMDI1MDkyOTgAQgh0aGlua2luZxIM1xfLptxSF8rKzou8GgwRLuVmfiBYOT4CZIsiMAYz8TpvchMAWodP+T9uZ+dJ47MVdNv+p86nZXmT5d5EWlCroiAOFHHzJH2o0UjJQSquAX6wu+jIC8/DXwurGIIf2ToldPuJWF5HGj8kHMnwh777XS1NUFl1irM24rvP7kTAv3Dp8YEwb8t3g1EsicY7j+C5DzgwxVsPYCVOCgdMqQjtsqpL7AX5KQ9+hR1gK1DQjshv4Gze3hw/+C+9iptd3RlDIuwO0cm8NY+sVqeWVh3TtQujQlVt1rW/HfWjclrtT0LZSGwc6RypmfMMQi6lncIk/EPgy0XbS1plKW0tLBgB", - "type": "thinking" - }, - { - "text": "PROMPT_DONE", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "anthropic-messages", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "claude-sonnet-4-5-20250929", - "flue.provider": "anthropic", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "claude-sonnet-4-5-20250929", - "provider": "anthropic" - } - metrics: { - "completion_tokens": 51, - "duration_ms": 0, - "estimated_cost": 0.0023391, - "prompt_cache_creation_tokens": 194, - "prompt_cached_tokens": 2692, - "prompt_tokens": 13, - "tokens": 2950 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json deleted file mode 100644 index e26112b02..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.json +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [ - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 113, - "duration_ms": 0, - "estimated_cost": 0.00031225000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 855, - "tokens": 968 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "output": { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 94, - "duration_ms": 0, - "estimated_cost": 0.00031790000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1002, - "tokens": 1096 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 87, - "duration_ms": 0, - "estimated_cost": 0.00034815000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1197, - "tokens": 1284 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 58, - "duration_ms": 0, - "estimated_cost": 0.0003321, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1298, - "tokens": 1356 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 7, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.00035900000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1370, - "tokens": 1438 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 9, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00043885000000000007, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1513, - "tokens": 1622 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "details": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 11, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 64, - "duration_ms": 0, - "estimated_cost": 0.00015592, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 226, - "tokens": 1826 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 13, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 90, - "duration_ms": 0, - "estimated_cost": 0.00021242000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 346, - "tokens": 1972 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "details": { - "command": "cat ./package.json", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 15, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 59, - "duration_ms": 0, - "estimated_cost": 0.0004814700000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 1885, - "tokens": 3480 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "command": "printf SKILL_DONE", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 17, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00031035000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 2560, - "prompt_tokens": 1252, - "tokens": 3819 - } - } - ], - "input": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "output": "SKILL_DONE", - "metadata": { - "flue.session": "skill", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00030565000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 847, - "tokens": 956 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "cwd": ".", - "messageId": "", - "session": "task:skill:", - "taskId": "" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00020255000000000002, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 969, - "tokens": 976 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.0001263, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 826, - "tokens": 830 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", - "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 19, - "duration_ms": 0, - "estimated_cost": 0.00022475000000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1005, - "tokens": 1024 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 34, - "duration_ms": 0, - "estimated_cost": 0.00025350000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1055, - "tokens": 1089 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 30, - "duration_ms": 0, - "estimated_cost": 0.00026690000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1147, - "tokens": 1177 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt deleted file mode 100644 index abf057e06..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-auto-hook.span-tree.txt +++ /dev/null @@ -1,1499 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "task", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 109, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00030565000000000003, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 0, -│ │ │ "prompt_tokens": 847, -│ │ │ "tokens": 956 -│ │ │ } -│ │ │ └── flue.task [task] -│ │ │ input: "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ output: "SKILL_DONE" -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "user" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_mode": "full", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 113, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031225000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 855, -│ │ │ │ "tokens": 968 -│ │ │ │ } -│ │ │ ├── tool:task [tool] -│ │ │ │ input: { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": {} -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "task", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": true, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "task" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 1, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 94, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031790000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1002, -│ │ │ │ "tokens": 1096 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 3, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 87, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00034815000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1197, -│ │ │ │ "tokens": 1284 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 5, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 58, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0003321, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1298, -│ │ │ │ "tokens": 1356 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 7, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 68, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00035900000000000005, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1370, -│ │ │ │ "tokens": 1438 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 9, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 109, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00043885000000000007, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1513, -│ │ │ │ "tokens": 1622 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 11, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 64, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00015592, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 226, -│ │ │ │ "tokens": 1826 -│ │ │ │ } -│ │ │ ├── tool:read [tool] -│ │ │ │ input: { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "lines": 7, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "read", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "read" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 13, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 90, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00021242000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 346, -│ │ │ │ "tokens": 1972 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 15, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 59, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0004814700000000001, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 1885, -│ │ │ │ "tokens": 3480 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "SKILL_DONE", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ └── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "command": "printf SKILL_DONE", -│ │ │ "timeout": 100000 -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "bash", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "bash" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.input_message_offset": 17, -│ │ │ "flue.input_mode": "delta", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "task:skill:", -│ │ │ "flue.stop_reason": "stop", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 7, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00031035000000000004, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 2560, -│ │ │ "prompt_tokens": 1252, -│ │ │ "tokens": 3819 -│ │ │ } -│ │ ├── tool:task [tool] -│ │ │ input: { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "cwd": ".", -│ │ │ "messageId": "", -│ │ │ "session": "task:skill:", -│ │ │ "taskId": "" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "task", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "cwd": ".", -│ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ }, -│ │ "id": "", -│ │ "name": "task", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "task" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.model": "gpt-5.4-nano", -│ │ "flue.provider": "openai", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-5.4-nano", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 7, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00020255000000000002, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 969, -│ │ "tokens": 976 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.0001263, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 826, -│ │ "tokens": 830 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", -│ "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_mode": "full", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "duration_ms": 0, - │ "estimated_cost": 0.00022475000000000001, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1005, - │ "tokens": 1024 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 34, - │ "duration_ms": 0, - │ "estimated_cost": 0.00025350000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1055, - │ "tokens": 1089 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 30, - │ "duration_ms": 0, - │ "estimated_cost": 0.00026690000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1147, - │ "tokens": 1177 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - } - metrics: { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json deleted file mode 100644 index e26112b02..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.json +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [ - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 113, - "duration_ms": 0, - "estimated_cost": 0.00031225000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 855, - "tokens": 968 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "output": { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 94, - "duration_ms": 0, - "estimated_cost": 0.00031790000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1002, - "tokens": 1096 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 87, - "duration_ms": 0, - "estimated_cost": 0.00034815000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1197, - "tokens": 1284 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 58, - "duration_ms": 0, - "estimated_cost": 0.0003321, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1298, - "tokens": 1356 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 7, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.00035900000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1370, - "tokens": 1438 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 9, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00043885000000000007, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1513, - "tokens": 1622 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "details": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 11, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 64, - "duration_ms": 0, - "estimated_cost": 0.00015592, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 226, - "tokens": 1826 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 13, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 90, - "duration_ms": 0, - "estimated_cost": 0.00021242000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 346, - "tokens": 1972 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "details": { - "command": "cat ./package.json", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 15, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 59, - "duration_ms": 0, - "estimated_cost": 0.0004814700000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 1885, - "tokens": 3480 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "command": "printf SKILL_DONE", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 17, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00031035000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 2560, - "prompt_tokens": 1252, - "tokens": 3819 - } - } - ], - "input": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "output": "SKILL_DONE", - "metadata": { - "flue.session": "skill", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00030565000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 847, - "tokens": 956 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "cwd": ".", - "messageId": "", - "session": "task:skill:", - "taskId": "" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00020255000000000002, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 969, - "tokens": 976 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.0001263, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 826, - "tokens": 830 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", - "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 19, - "duration_ms": 0, - "estimated_cost": 0.00022475000000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1005, - "tokens": 1024 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 34, - "duration_ms": 0, - "estimated_cost": 0.00025350000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1055, - "tokens": 1089 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 30, - "duration_ms": 0, - "estimated_cost": 0.00026690000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1147, - "tokens": 1177 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt deleted file mode 100644 index abf057e06..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-cli.span-tree.txt +++ /dev/null @@ -1,1499 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "task", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 109, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00030565000000000003, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 0, -│ │ │ "prompt_tokens": 847, -│ │ │ "tokens": 956 -│ │ │ } -│ │ │ └── flue.task [task] -│ │ │ input: "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ output: "SKILL_DONE" -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "user" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_mode": "full", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 113, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031225000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 855, -│ │ │ │ "tokens": 968 -│ │ │ │ } -│ │ │ ├── tool:task [tool] -│ │ │ │ input: { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": {} -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "task", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": true, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "task" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 1, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 94, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031790000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1002, -│ │ │ │ "tokens": 1096 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 3, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 87, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00034815000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1197, -│ │ │ │ "tokens": 1284 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 5, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 58, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0003321, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1298, -│ │ │ │ "tokens": 1356 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 7, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 68, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00035900000000000005, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1370, -│ │ │ │ "tokens": 1438 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 9, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 109, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00043885000000000007, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1513, -│ │ │ │ "tokens": 1622 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 11, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 64, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00015592, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 226, -│ │ │ │ "tokens": 1826 -│ │ │ │ } -│ │ │ ├── tool:read [tool] -│ │ │ │ input: { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "lines": 7, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "read", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "read" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 13, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 90, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00021242000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 346, -│ │ │ │ "tokens": 1972 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 15, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 59, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0004814700000000001, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 1885, -│ │ │ │ "tokens": 3480 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "SKILL_DONE", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ └── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "command": "printf SKILL_DONE", -│ │ │ "timeout": 100000 -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "bash", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "bash" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.input_message_offset": 17, -│ │ │ "flue.input_mode": "delta", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "task:skill:", -│ │ │ "flue.stop_reason": "stop", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 7, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00031035000000000004, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 2560, -│ │ │ "prompt_tokens": 1252, -│ │ │ "tokens": 3819 -│ │ │ } -│ │ ├── tool:task [tool] -│ │ │ input: { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "cwd": ".", -│ │ │ "messageId": "", -│ │ │ "session": "task:skill:", -│ │ │ "taskId": "" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "task", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "cwd": ".", -│ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ }, -│ │ "id": "", -│ │ "name": "task", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "task" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.model": "gpt-5.4-nano", -│ │ "flue.provider": "openai", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-5.4-nano", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 7, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00020255000000000002, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 969, -│ │ "tokens": 976 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.0001263, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 826, -│ │ "tokens": 830 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", -│ "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_mode": "full", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "duration_ms": 0, - │ "estimated_cost": 0.00022475000000000001, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1005, - │ "tokens": 1024 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 34, - │ "duration_ms": 0, - │ "estimated_cost": 0.00025350000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1055, - │ "tokens": 1089 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 30, - │ "duration_ms": 0, - │ "estimated_cost": 0.00026690000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1147, - │ "tokens": 1177 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - } - metrics: { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json deleted file mode 100644 index e26112b02..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.json +++ /dev/null @@ -1,1662 +0,0 @@ -{ - "span_tree": [ - { - "name": "workflow:instrumentation", - "type": "task", - "children": [ - { - "name": "flue.skill", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [ - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 113, - "duration_ms": 0, - "estimated_cost": 0.00031225000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 855, - "tokens": 968 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "output": { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "details": {} - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - }, - "error": "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "agent": "e2e-flue-skill", - "cwd": ".", - "description": "Run e2e-flue-skill with marker SKILL_DONE", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", - "type": "text" - } - ], - "isError": true, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 94, - "duration_ms": 0, - "estimated_cost": 0.00031790000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1002, - "tokens": 1096 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -name AGENTS.md -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 87, - "duration_ms": 0, - "estimated_cost": 0.00034815000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1197, - "tokens": 1284 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 58, - "duration_ms": 0, - "estimated_cost": 0.0003321, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1298, - "tokens": 1356 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "(no output)", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 7, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 68, - "duration_ms": 0, - "estimated_cost": 0.00035900000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1370, - "tokens": 1438 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "details": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 9, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00043885000000000007, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1513, - "tokens": 1622 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "details": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "./.agents/skills/e2e-flue-skill/SKILL.md", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 11, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 64, - "duration_ms": 0, - "estimated_cost": 0.00015592, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 226, - "tokens": 1826 - } - }, - { - "name": "tool:read", - "type": "tool", - "children": [], - "input": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "output": { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "details": { - "lines": 7, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "read", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "limit": 200, - "offset": 1, - "path": "./.agents/skills/e2e-flue-skill/SKILL.md" - }, - "id": "", - "name": "read", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "read" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 13, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 90, - "duration_ms": 0, - "estimated_cost": 0.00021242000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 346, - "tokens": 1972 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "details": { - "command": "cat ./package.json", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "cat ./package.json", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 15, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 59, - "duration_ms": 0, - "estimated_cost": 0.0004814700000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 1536, - "prompt_tokens": 1885, - "tokens": 3480 - } - }, - { - "name": "tool:bash", - "type": "tool", - "children": [], - "input": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "command": "printf SKILL_DONE", - "exitCode": 0 - } - }, - "metadata": { - "flue.session": "task:skill:", - "flue.tool_name": "bash", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "command": "printf SKILL_DONE", - "timeout": 100000 - }, - "id": "", - "name": "bash", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "bash" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 17, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "task:skill:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00031035000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 2560, - "prompt_tokens": 1252, - "tokens": 3819 - } - } - ], - "input": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", - "output": "SKILL_DONE", - "metadata": { - "flue.session": "skill", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 109, - "duration_ms": 0, - "estimated_cost": 0.00030565000000000003, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 847, - "tokens": 956 - } - }, - { - "name": "tool:task", - "type": "tool", - "children": [], - "input": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "output": { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "details": { - "cwd": ".", - "messageId": "", - "session": "task:skill:", - "taskId": "" - } - }, - "metadata": { - "flue.session": "skill", - "flue.tool_name": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - }, - { - "content": [ - { - "arguments": { - "cwd": ".", - "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", - "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." - }, - "id": "", - "name": "task", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "SKILL_DONE", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "task" - } - ], - "output": { - "content": [ - { - "text": "SKILL_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "skill", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00020255000000000002, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 969, - "tokens": 976 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "SKILL_DONE", - "metadata": { - "flue.operation": "skill", - "flue.session": "skill", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.task", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Reply with exactly TASK_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "TASK_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "task:task:", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 4, - "duration_ms": 0, - "estimated_cost": 0.0001263, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 826, - "tokens": 830 - } - } - ], - "input": "Reply with exactly TASK_DONE and no other text.", - "output": "TASK_DONE", - "metadata": { - "flue.session": "task", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.compact", - "type": "task", - "children": [ - { - "name": "compaction:manual", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", - "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.model": "gpt-4o-mini", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "compaction_prefix", - "model": "gpt-4o-mini", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 0, - "duration_ms": 0, - "estimated_cost": 0, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 0, - "tokens": 0 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "messagesAfter": 2, - "messagesBefore": 8 - }, - "metadata": { - "flue.compaction_reason": "manual", - "flue.session": "main", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0, - "messages_after": 2, - "messages_before": 8 - } - } - ], - "input": { - "estimatedTokens": 1229, - "reason": "manual" - }, - "output": { - "completed": true - }, - "metadata": { - "flue.operation": "compact", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ], - "input": { - "metadata": { - "scenario": "flue-instrumentation", - "testRunId": "" - }, - "scenario": "flue-instrumentation" - }, - "output": { - "scenario": "flue-instrumentation", - "status": "done" - }, - "metadata": { - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.prompt", - "type": "task", - "children": [ - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_mode": "full", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 19, - "duration_ms": 0, - "estimated_cost": 0.00022475000000000001, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1005, - "tokens": 1024 - } - }, - { - "name": "tool:lookup", - "type": "tool", - "children": [], - "input": { - "query": "flue instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "details": { - "customTool": "lookup" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "lookup", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "query": "flue instrumentation" - }, - "id": "", - "name": "lookup", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "lookup" - } - ], - "output": { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 1, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 34, - "duration_ms": 0, - "estimated_cost": 0.00025350000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1055, - "tokens": 1089 - } - }, - { - "name": "tool:web_search", - "type": "tool", - "children": [], - "input": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "output": { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "details": { - "customTool": "web_search" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "web_search", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "lookupId": "flue-session-2026", - "query": "Braintrust Flue reasoning stream instrumentation" - }, - "id": "", - "name": "web_search", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "web_search" - } - ], - "output": { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 3, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "toolUse", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 30, - "duration_ms": 0, - "estimated_cost": 0.00026690000000000004, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1147, - "tokens": 1177 - } - }, - { - "name": "tool:summarize_source", - "type": "tool", - "children": [], - "input": { - "url": "https://example.test/flue/reasoning-streams" - }, - "output": { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "details": { - "customTool": "summarize_source" - } - }, - "metadata": { - "flue.session": "main", - "flue.tool_name": "summarize_source", - "provider": "flue" - }, - "metrics": { - "duration_ms": 0 - } - }, - { - "name": "flue.turn", - "type": "llm", - "children": [], - "input": [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ], - "output": { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - }, - "metadata": { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - }, - "metrics": { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } - } - ], - "input": [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ], - "output": "PROMPT_DONE", - "metadata": { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - }, - "metrics": { - "duration_ms": 0 - } - } - ] -} diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt deleted file mode 100644 index abf057e06..000000000 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v0-8-latest-explicit.span-tree.txt +++ /dev/null @@ -1,1499 +0,0 @@ -span_tree: -├── workflow:instrumentation [task] -│ input: { -│ "metadata": { -│ "scenario": "flue-instrumentation", -│ "testRunId": "" -│ }, -│ "scenario": "flue-instrumentation" -│ } -│ output: { -│ "scenario": "flue-instrumentation", -│ "status": "done" -│ } -│ metadata: { -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ ├── flue.skill [task] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: "SKILL_DONE" -│ │ metadata: { -│ │ "flue.operation": "skill", -│ │ "flue.session": "skill", -│ │ "flue.workflow_name": "instrumentation", -│ │ "provider": "flue", -│ │ "scenario": "flue-instrumentation" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ ├── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "user" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "task", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "skill", -│ │ │ "flue.stop_reason": "toolUse", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 109, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00030565000000000003, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 0, -│ │ │ "prompt_tokens": 847, -│ │ │ "tokens": 956 -│ │ │ } -│ │ │ └── flue.task [task] -│ │ │ input: "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ output: "SKILL_DONE" -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished.", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "user" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_mode": "full", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 113, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031225000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 855, -│ │ │ │ "tokens": 968 -│ │ │ │ } -│ │ │ ├── tool:task [tool] -│ │ │ │ input: { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": {} -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "task", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ │ error: "{\"content\":[{\"type\":\"text\",\"text\":\"[flue] Subagent \\\"e2e-flue-skill\\\" is not declared. Available: (none).\"}],\"details\":{}}" -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "agent": "e2e-flue-skill", -│ │ │ │ "cwd": ".", -│ │ │ │ "description": "Run e2e-flue-skill with marker SKILL_DONE", -│ │ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Ensure completion and report the final output marker only." -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "task", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "[flue] Subagent \"e2e-flue-skill\" is not declared. Available: (none).", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": true, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "task" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 1, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 94, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00031790000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1002, -│ │ │ │ "tokens": 1096 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -name AGENTS.md -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 3, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 87, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00034815000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1197, -│ │ │ │ "tokens": 1284 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 4 -type f -name AGENTS.md -print -exec sed -n '1,200p' {} \\;", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 5, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 58, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0003321, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1298, -│ │ │ │ "tokens": 1356 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 6 -type f -name 'AGENTS.md' -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "(no output)", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 7, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 68, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00035900000000000005, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1370, -│ │ │ │ "tokens": 1438 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find . -maxdepth 5 -type f -name '*flue*skill*' -o -name '*skill*' | head", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills\n./.agents/skills/e2e-flue-skill", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 9, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 109, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00043885000000000007, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 0, -│ │ │ │ "prompt_tokens": 1513, -│ │ │ │ "tokens": 1622 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "find ./.agents/skills/e2e-flue-skill -maxdepth 2 -type f -print", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "./.agents/skills/e2e-flue-skill/SKILL.md", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 11, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 64, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00015592, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 226, -│ │ │ │ "tokens": 1826 -│ │ │ │ } -│ │ │ ├── tool:read [tool] -│ │ │ │ input: { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "lines": 7, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "read", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "limit": 200, -│ │ │ │ "offset": 1, -│ │ │ │ "path": "./.agents/skills/e2e-flue-skill/SKILL.md" -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "read", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "---\nname: e2e-flue-skill\ndescription: Deterministic skill used by the Braintrust Flue instrumentation e2e test.\n---\n\nReturn the marker from the provided args exactly once. Output no other text.\n", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "read" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 13, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 90, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.00021242000000000003, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 346, -│ │ │ │ "tokens": 1972 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ ├── flue.turn [llm] -│ │ │ │ input: [ -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "cat ./package.json", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ }, -│ │ │ │ { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "{\n \"name\": \"flue-instrumentation-scenario\",\n \"private\": true,\n \"type\": \"module\",\n \"braintrustScenario\": {\n \"bump\": {\n \"dependencies\": {\n \"flue-cli-v0-8-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-cli-v1-latest\": {\n \"package\": \"@flue/cli\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v0-8-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=0.8.0 <0.9.0\"\n },\n \"flue-runtime-v1-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=1.0.0-beta.3 <2.0.0-0\",\n \"allowPrerelease\": true\n },\n \"flue-runtime-v2-latest\": {\n \"package\": \"@flue/runtime\",\n \"range\": \">=2.0.0 <3.0.0\"\n }\n }\n }\n },\n \"dependencies\": {\n \"@flue/cli\": \"0.8.0\",\n \"@flue/runtime\": \"0.8.0\",\n \"flue-cli-v0-8-latest\": \"npm:@flue/cli@0.8.1\",\n \"flue-cli-v1\": \"npm:@flue/cli@1.0.0-beta.3\",\n \"flue-cli-v1-latest\": \"npm:@flue/cli@1.0.0-beta.9\",\n \"flue-runtime-v0-8-latest\": \"npm:@flue/runtime@0.8.1\",\n \"flue-runtime-v1\": \"npm:@flue/runtime@1.0.0-beta.3\",\n \"flue-runtime-v1-latest\": \"npm:@flue/runtime@1.0.0-beta.9\",\n \"flue-runtime-v2\": \"npm:@flue/runtime@2.0.0\",\n \"flue-runtime-v2-latest\": \"npm:@flue/runtime@2.0.1\",\n \"pi-ai-v2\": \"npm:@earendil-works/pi-ai@0.83.0\",\n \"valibot\": \"1.4.2\"\n },\n \"pnpm\": {\n \"overrides\": {\n \"@flue/runtime@0.8.0>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.0>@earendil-works/pi-ai\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-agent-core\": \"0.75.4\",\n \"@flue/runtime@0.8.1>@earendil-works/pi-ai\": \"0.75.4\",\n \"@hono/node-server@>=2.0.0 <2.0.10\": \"^2.0.10\",\n \"fast-uri@<3.1.5\": \"^3.1.5\",\n \"hono@>=4.0.0 <4.12.27\": \"^4.12.27\",\n \"brace-expansion@>=3.0.0 <5.0.9\": \"^5.0.9\",\n \"postcss@<8.5.18\": \"^8.5.18\",\n \"valibot@<1.4.2\": \"^1.4.2\",\n \"ip-address@<10.2.2\": \"^10.2.2\",\n \"undici@<7.29.0\": \"^7.29.0\"\n }\n }\n}", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "isError": false, -│ │ │ │ "role": "toolResult", -│ │ │ │ "toolCallId": "", -│ │ │ │ "toolName": "bash" -│ │ │ │ } -│ │ │ │ ] -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "arguments": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ }, -│ │ │ │ "id": "", -│ │ │ │ "name": "bash", -│ │ │ │ "type": "toolCall" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "role": "assistant" -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.api": "openai-responses", -│ │ │ │ "flue.input_message_offset": 15, -│ │ │ │ "flue.input_mode": "delta", -│ │ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ │ "flue.provider": "openai", -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.stop_reason": "toolUse", -│ │ │ │ "flue.turn_purpose": "agent", -│ │ │ │ "model": "gpt-5.4-nano", -│ │ │ │ "provider": "openai" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "completion_tokens": 59, -│ │ │ │ "duration_ms": 0, -│ │ │ │ "estimated_cost": 0.0004814700000000001, -│ │ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ │ "prompt_cached_tokens": 1536, -│ │ │ │ "prompt_tokens": 1885, -│ │ │ │ "tokens": 3480 -│ │ │ │ } -│ │ │ ├── tool:bash [tool] -│ │ │ │ input: { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "timeout": 100000 -│ │ │ │ } -│ │ │ │ output: { -│ │ │ │ "content": [ -│ │ │ │ { -│ │ │ │ "text": "SKILL_DONE", -│ │ │ │ "type": "text" -│ │ │ │ } -│ │ │ │ ], -│ │ │ │ "details": { -│ │ │ │ "command": "printf SKILL_DONE", -│ │ │ │ "exitCode": 0 -│ │ │ │ } -│ │ │ │ } -│ │ │ │ metadata: { -│ │ │ │ "flue.session": "task:skill:", -│ │ │ │ "flue.tool_name": "bash", -│ │ │ │ "provider": "flue" -│ │ │ │ } -│ │ │ │ metrics: { -│ │ │ │ "duration_ms": 0 -│ │ │ │ } -│ │ │ └── flue.turn [llm] -│ │ │ input: [ -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "arguments": { -│ │ │ "command": "printf SKILL_DONE", -│ │ │ "timeout": 100000 -│ │ │ }, -│ │ │ "id": "", -│ │ │ "name": "bash", -│ │ │ "type": "toolCall" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ }, -│ │ │ { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "isError": false, -│ │ │ "role": "toolResult", -│ │ │ "toolCallId": "", -│ │ │ "toolName": "bash" -│ │ │ } -│ │ │ ] -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "textSignature": "{\"v\":1,\"id\":\"msg_054a16a2f2ea3217016a705b70a41481a38ece40ecddf69512\",\"phase\":\"final_answer\"}", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "role": "assistant" -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.api": "openai-responses", -│ │ │ "flue.input_message_offset": 17, -│ │ │ "flue.input_mode": "delta", -│ │ │ "flue.model": "gpt-5.4-nano", -│ │ │ "flue.provider": "openai", -│ │ │ "flue.session": "task:skill:", -│ │ │ "flue.stop_reason": "stop", -│ │ │ "flue.turn_purpose": "agent", -│ │ │ "model": "gpt-5.4-nano", -│ │ │ "provider": "openai" -│ │ │ } -│ │ │ metrics: { -│ │ │ "completion_tokens": 7, -│ │ │ "duration_ms": 0, -│ │ │ "estimated_cost": 0.00031035000000000004, -│ │ │ "prompt_cache_creation_tokens": 0, -│ │ │ "prompt_cached_tokens": 2560, -│ │ │ "prompt_tokens": 1252, -│ │ │ "tokens": 3819 -│ │ │ } -│ │ ├── tool:task [tool] -│ │ │ input: { -│ │ │ "cwd": ".", -│ │ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ │ } -│ │ │ output: { -│ │ │ "content": [ -│ │ │ { -│ │ │ "text": "SKILL_DONE", -│ │ │ "type": "text" -│ │ │ } -│ │ │ ], -│ │ │ "details": { -│ │ │ "cwd": ".", -│ │ │ "messageId": "", -│ │ │ "session": "task:skill:", -│ │ │ "taskId": "" -│ │ │ } -│ │ │ } -│ │ │ metadata: { -│ │ │ "flue.session": "skill", -│ │ │ "flue.tool_name": "task", -│ │ │ "provider": "flue" -│ │ │ } -│ │ │ metrics: { -│ │ │ "duration_ms": 0 -│ │ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\nArguments:\n{\n \"marker\": \"SKILL_DONE\"\n}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "arguments": { -│ │ "cwd": ".", -│ │ "description": "Run the deterministic Flue instrumentation e2e test skill e2e-flue-skill.", -│ │ "prompt": "Run the skill named \"e2e-flue-skill\" with arguments {\"marker\":\"SKILL_DONE\"}. Output only the marker and no extra text when finished." -│ │ }, -│ │ "id": "", -│ │ "name": "task", -│ │ "type": "toolCall" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ }, -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "isError": false, -│ │ "role": "toolResult", -│ │ "toolCallId": "", -│ │ "toolName": "task" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "SKILL_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f099b3fce0721fd016a705b71558881a38c53a767e09d9e31\",\"phase\":\"final_answer\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.model": "gpt-5.4-nano", -│ │ "flue.provider": "openai", -│ │ "flue.session": "skill", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-5.4-nano", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 7, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.00020255000000000002, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 969, -│ │ "tokens": 976 -│ │ } -│ ├── flue.task [task] -│ │ input: "Reply with exactly TASK_DONE and no other text." -│ │ output: "TASK_DONE" -│ │ metadata: { -│ │ "flue.session": "task", -│ │ "provider": "flue" -│ │ } -│ │ metrics: { -│ │ "duration_ms": 0 -│ │ } -│ │ └── flue.turn [llm] -│ │ input: [ -│ │ { -│ │ "content": [ -│ │ { -│ │ "text": "Reply with exactly TASK_DONE and no other text.", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "user" -│ │ } -│ │ ] -│ │ output: { -│ │ "content": [ -│ │ { -│ │ "text": "TASK_DONE", -│ │ "textSignature": "{\"v\":1,\"id\":\"msg_0f3adfdb1510c165016a705b72843c81a0b30d242da683a882\"}", -│ │ "type": "text" -│ │ } -│ │ ], -│ │ "role": "assistant" -│ │ } -│ │ metadata: { -│ │ "flue.api": "openai-responses", -│ │ "flue.input_mode": "full", -│ │ "flue.model": "gpt-4o-mini", -│ │ "flue.provider": "openai", -│ │ "flue.session": "task:task:", -│ │ "flue.stop_reason": "stop", -│ │ "flue.turn_purpose": "agent", -│ │ "model": "gpt-4o-mini", -│ │ "provider": "openai" -│ │ } -│ │ metrics: { -│ │ "completion_tokens": 4, -│ │ "duration_ms": 0, -│ │ "estimated_cost": 0.0001263, -│ │ "prompt_cache_creation_tokens": 0, -│ │ "prompt_cached_tokens": 0, -│ │ "prompt_tokens": 826, -│ │ "tokens": 830 -│ │ } -│ └── flue.compact [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "completed": true -│ } -│ metadata: { -│ "flue.operation": "compact", -│ "flue.session": "main", -│ "flue.workflow_name": "instrumentation", -│ "provider": "flue", -│ "scenario": "flue-instrumentation" -│ } -│ metrics: { -│ "duration_ms": 0 -│ } -│ └── compaction:manual [task] -│ input: { -│ "estimatedTokens": 1229, -│ "reason": "manual" -│ } -│ output: { -│ "messagesAfter": 2, -│ "messagesBefore": 8 -│ } -│ metadata: { -│ "flue.compaction_reason": "manual", -│ "flue.session": "main", -│ "provider": "flue" -│ } -│ metrics: { -│ "duration_ms": 0, -│ "messages_after": 2, -│ "messages_before": 8 -│ } -│ └── flue.turn [llm] -│ input: [ -│ { -│ "content": [ -│ { -│ "text": "\n[User]: Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.\n\n[Assistant tool calls]: lookup(query=\"flue instrumentation\")\n\n[Tool result]: {\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}\n\n[Assistant tool calls]: web_search(lookupId=\"flue-session-2026\", query=\"Braintrust Flue reasoning stream instrumentation\")\n\n[Tool result]: {\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}\n\n[Assistant tool calls]: summarize_source(url=\"https://example.test/flue/reasoning-streams\")\n\n[Tool result]: {\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}\n\n\nThis is the PREFIX of a turn that was too large to keep. The SUFFIX (recent work) is retained.\n\nSummarize the prefix to provide context for the retained suffix:\n\n## Original Request\n[What did the user ask for in this turn?]\n\n## Early Progress\n- [Key decisions and work done in the prefix]\n\n## Context for Suffix\n- [Information needed to understand the retained recent work]\n\nBe concise. Focus on what's needed to understand the kept suffix.", -│ "type": "text" -│ } -│ ], -│ "role": "user" -│ } -│ ] -│ output: { -│ "content": [ -│ { -│ "text": "## Original Request\nThe user requested a step-by-step execution of an instrumented research flow involving tool calls to gather information on \"flue instrumentation\" and \"", -│ "textSignature": "{\"v\":1,\"id\":\"msg_04d28ba03840bb65016a705b7358a081a38e69464057840036\"}", -│ "type": "text" -│ } -│ ], -│ "role": "assistant" -│ } -│ metadata: { -│ "flue.api": "openai-responses", -│ "flue.model": "gpt-4o-mini", -│ "flue.provider": "openai", -│ "flue.session": "main", -│ "flue.stop_reason": "stop", -│ "flue.turn_purpose": "compaction_prefix", -│ "model": "gpt-4o-mini", -│ "provider": "openai" -│ } -│ metrics: { -│ "completion_tokens": 0, -│ "duration_ms": 0, -│ "estimated_cost": 0, -│ "prompt_cache_creation_tokens": 0, -│ "prompt_cached_tokens": 0, -│ "prompt_tokens": 0, -│ "tokens": 0 -│ } -└── flue.prompt [task] - input: [ - { - "content": [ - { - "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - "type": "text" - } - ], - "role": "user" - } - ] - output: "PROMPT_DONE" - metadata: { - "flue.operation": "prompt", - "flue.session": "main", - "flue.workflow_name": "instrumentation", - "provider": "flue", - "scenario": "flue-instrumentation" - } - metrics: { - "duration_ms": 0 - } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "text": "Complete this instrumented research flow. Call exactly one tool per turn and wait for each tool result before choosing the next tool. Step 1: call lookup with query \"flue instrumentation\". Step 2: use the lookup result id as lookupId and call web_search with query \"Braintrust Flue reasoning stream instrumentation\". Step 3: use the first web_search result url and call summarize_source. After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - │ "type": "text" - │ } - │ ], - │ "role": "user" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_mode": "full", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 19, - │ "duration_ms": 0, - │ "estimated_cost": 0.00022475000000000001, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1005, - │ "tokens": 1024 - │ } - ├── tool:lookup [tool] - │ input: { - │ "query": "flue instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "lookup" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "lookup", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "query": "flue instrumentation" - │ }, - │ "id": "", - │ "name": "lookup", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"id\":\"flue-session-2026\",\"query\":\"flue instrumentation\",\"topic\":\"session instrumentation\"}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "lookup" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 1, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 34, - │ "duration_ms": 0, - │ "estimated_cost": 0.00025350000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1055, - │ "tokens": 1089 - │ } - ├── tool:web_search [tool] - │ input: { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "web_search" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "web_search", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - ├── flue.turn [llm] - │ input: [ - │ { - │ "content": [ - │ { - │ "arguments": { - │ "lookupId": "flue-session-2026", - │ "query": "Braintrust Flue reasoning stream instrumentation" - │ }, - │ "id": "", - │ "name": "web_search", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ }, - │ { - │ "content": [ - │ { - │ "text": "{\"lookupId\":\"flue-session-2026\",\"query\":\"Braintrust Flue reasoning stream instrumentation\",\"results\":[{\"title\":\"Flue reasoning stream instrumentation\",\"url\":\"https://example.test/flue/reasoning-streams\"}]}", - │ "type": "text" - │ } - │ ], - │ "isError": false, - │ "role": "toolResult", - │ "toolCallId": "", - │ "toolName": "web_search" - │ } - │ ] - │ output: { - │ "content": [ - │ { - │ "arguments": { - │ "url": "https://example.test/flue/reasoning-streams" - │ }, - │ "id": "", - │ "name": "summarize_source", - │ "type": "toolCall" - │ } - │ ], - │ "role": "assistant" - │ } - │ metadata: { - │ "flue.api": "openai-responses", - │ "flue.input_message_offset": 3, - │ "flue.input_mode": "delta", - │ "flue.model": "gpt-5.4-nano", - │ "flue.provider": "openai", - │ "flue.session": "main", - │ "flue.stop_reason": "toolUse", - │ "flue.turn_purpose": "agent", - │ "model": "gpt-5.4-nano", - │ "provider": "openai" - │ } - │ metrics: { - │ "completion_tokens": 30, - │ "duration_ms": 0, - │ "estimated_cost": 0.00026690000000000004, - │ "prompt_cache_creation_tokens": 0, - │ "prompt_cached_tokens": 0, - │ "prompt_tokens": 1147, - │ "tokens": 1177 - │ } - ├── tool:summarize_source [tool] - │ input: { - │ "url": "https://example.test/flue/reasoning-streams" - │ } - │ output: { - │ "content": [ - │ { - │ "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - │ "type": "text" - │ } - │ ], - │ "details": { - │ "customTool": "summarize_source" - │ } - │ } - │ metadata: { - │ "flue.session": "main", - │ "flue.tool_name": "summarize_source", - │ "provider": "flue" - │ } - │ metrics: { - │ "duration_ms": 0 - │ } - └── flue.turn [llm] - input: [ - { - "content": [ - { - "arguments": { - "url": "https://example.test/flue/reasoning-streams" - }, - "id": "", - "name": "summarize_source", - "type": "toolCall" - } - ], - "role": "assistant" - }, - { - "content": [ - { - "text": "{\"summary\":\"Flue emits reasoning, tool execution, and LLM turn events separately.\",\"url\":\"https://example.test/flue/reasoning-streams\"}", - "type": "text" - } - ], - "isError": false, - "role": "toolResult", - "toolCallId": "", - "toolName": "summarize_source" - } - ] - output: { - "content": [ - { - "text": "PROMPT_DONE", - "textSignature": "{\"v\":1,\"id\":\"msg_045458052dc10379016a705b62dbd8819189b8287061debf12\",\"phase\":\"final_answer\"}", - "type": "text" - } - ], - "role": "assistant" - } - metadata: { - "flue.api": "openai-responses", - "flue.input_message_offset": 5, - "flue.input_mode": "delta", - "flue.model": "gpt-5.4-nano", - "flue.provider": "openai", - "flue.session": "main", - "flue.stop_reason": "stop", - "flue.turn_purpose": "agent", - "model": "gpt-5.4-nano", - "provider": "openai" - } - metrics: { - "completion_tokens": 7, - "duration_ms": 0, - "estimated_cost": 0.00025315000000000005, - "prompt_cache_creation_tokens": 0, - "prompt_cached_tokens": 0, - "prompt_tokens": 1222, - "tokens": 1229 - } diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json index 464358cef..aa0fef2c7 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.json @@ -75,7 +75,7 @@ "input": { "name": "e2e-flue-skill" }, - "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "metadata": { "flue.session": "skill", "flue.tool_name": "activate_skill", @@ -119,7 +119,7 @@ { "content": [ { - "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "type": "text" } ], diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt index 104f75f29..60b2ac825 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-0-0-beta-3-explicit.span-tree.txt @@ -99,7 +99,7 @@ span_tree: │ │ │ input: { │ │ │ "name": "e2e-flue-skill" │ │ │ } -│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" +│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" │ │ │ metadata: { │ │ │ "flue.session": "skill", │ │ │ "flue.tool_name": "activate_skill", @@ -139,7 +139,7 @@ span_tree: │ │ { │ │ "content": [ │ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", +│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", │ │ "type": "text" │ │ } │ │ ], diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json index ac0646db1..b4336fbab 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.json @@ -71,7 +71,7 @@ "input": { "name": "e2e-flue-skill" }, - "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "output": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "metadata": { "flue.session": "skill", "flue.tool_name": "activate_skill", @@ -111,7 +111,7 @@ { "content": [ { - "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", + "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", "type": "text" } ], diff --git a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt index 2a17573af..dd492f42e 100644 --- a/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt +++ b/e2e/scenarios/flue-instrumentation/__snapshots__/flue-v1-latest-explicit.span-tree.txt @@ -95,7 +95,7 @@ span_tree: │ │ │ input: { │ │ │ "name": "e2e-flue-skill" │ │ │ } -│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" +│ │ │ output: "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n" │ │ │ metadata: { │ │ │ "flue.session": "skill", │ │ │ "flue.tool_name": "activate_skill", @@ -131,7 +131,7 @@ span_tree: │ │ { │ │ "content": [ │ │ { -│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-v1/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", +│ │ "text": "Run the skill named \"e2e-flue-skill\".\n\n\nReturn the marker from the provided args exactly once. Output no other text.\n\n\nSupporting skill resources are available relative to this workspace skill directory but are not loaded into context unless needed:\n\n- Base directory: /e2e/scenarios/flue-instrumentation/.agents/skills/e2e-flue-skill\n- Resolve relative resource paths from this directory and read only the files you need.\n", │ │ "type": "text" │ │ } │ │ ], diff --git a/e2e/scenarios/flue-instrumentation/assertions.ts b/e2e/scenarios/flue-instrumentation/assertions.ts index 53a597fbd..89b564836 100644 --- a/e2e/scenarios/flue-instrumentation/assertions.ts +++ b/e2e/scenarios/flue-instrumentation/assertions.ts @@ -20,14 +20,6 @@ import { import { SCENARIO_NAME } from "./constants.mjs"; type RunFlueScenario = (harness: { - runNodeScenarioDir: (options: { - entry: string; - env?: Record; - nodeArgs?: string[]; - runContext?: ScenarioRunContext; - scenarioDir: string; - timeoutMs: number; - }) => Promise; runScenarioDir: (options: { entry: string; env?: Record; @@ -155,40 +147,24 @@ function firstSpanIndex( return index === -1 ? Number.MAX_SAFE_INTEGER : index; } -function findFlueDescendants( - events: CapturedLogEvent[], - flueSpan: CapturedLogEvent | undefined, - predicate: (event: CapturedLogEvent) => boolean, -): CapturedLogEvent[] { - return findMatchingDescendants(events, flueSpan, predicate); -} - -function isFlueChildSpan( - event: CapturedLogEvent, - includeAmbientProbeSpans = true, -): boolean { +function isFlueChildSpan(event: CapturedLogEvent): boolean { return ( event.span.name === "flue.turn" || event.span.name?.startsWith("tool:") === true || event.span.name?.startsWith("task:") === true || event.span.name?.startsWith("compaction:") === true || - (includeAmbientProbeSpans && event.span.name === "flue.toolCurrentProbe") || + event.span.name === "flue.toolCurrentProbe" || event.span.name === "flue.task" ); } -function buildSpanTree( - events: CapturedLogEvent[], - includeAmbientProbeSpans: boolean, -): SpanTreeEntry[] { +function buildSpanTree(events: CapturedLogEvent[]): SpanTreeEntry[] { const workflow = findLatestSpanByPrefix(events, "workflow:"); - const workflowCurrentProbe = includeAmbientProbeSpans - ? findLatestChildSpan( - events, - "flue.workflowCurrentProbe", - workflow?.span.id, - ) - : undefined; + const workflowCurrentProbe = findLatestChildSpan( + events, + "flue.workflowCurrentProbe", + workflow?.span.id, + ); const promptSpan = findFlueOperation(events, "flue.prompt"); const skillSpan = findFlueOperation(events, "flue.skill"); const taskSpan = findFlueOperation(events, "flue.task"); @@ -198,28 +174,19 @@ function buildSpanTree( workflow, workflowCurrentProbe, promptSpan, - ...findFlueDescendants(events, promptSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, promptSpan, isFlueChildSpan), skillSpan, - ...findFlueDescendants(events, skillSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, skillSpan, isFlueChildSpan), taskSpan, - ...findFlueDescendants(events, taskSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, taskSpan, isFlueChildSpan), compactSpan, - ...findFlueDescendants(events, compactSpan, (event) => - isFlueChildSpan(event, includeAmbientProbeSpans), - ), + ...findMatchingDescendants(events, compactSpan, isFlueChildSpan), ].flatMap((event) => event ? [{ event, fields: snapshotFields(event) }] : [], ); } export function defineFlueInstrumentationAssertions(options: { - expectAmbientContext?: boolean; name: string; runScenario: RunFlueScenario; snapshotName: string; @@ -232,7 +199,6 @@ export function defineFlueInstrumentationAssertions(options: { ); const timeoutMs = effectiveScenarioTimeoutMs(options.timeoutMs); const testConfig = { timeout: timeoutMs }; - const expectAmbientContext = options.expectAmbientContext ?? true; describe.sequential(options.name, () => { let events: CapturedLogEvent[] = []; @@ -257,46 +223,42 @@ export function defineFlueInstrumentationAssertions(options: { }); }); - if (expectAmbientContext) { - test( - "makes the Flue workflow span current for app spans", - testConfig, - () => { - const workflow = findLatestSpanByPrefix(events, "workflow:"); - const appSpan = findLatestChildSpan( - events, - "flue.workflowCurrentProbe", - workflow?.span.id, - ); - - expect(appSpan).toBeDefined(); - expect(appSpan?.span.parentIds).toEqual([workflow?.span.id]); - expect(appSpan?.output).toBe("active"); - }, - ); - - test("makes Flue tool spans current for app spans", testConfig, () => { + test( + "makes the Flue workflow span current for app spans", + testConfig, + () => { const workflow = findLatestSpanByPrefix(events, "workflow:"); - const promptSpan = findFlueOperation(events, "flue.prompt"); - const lookupToolSpan = findFlueDescendants( - events, - promptSpan, - (event) => event.span.name === "tool:lookup", - )[0]; const appSpan = findLatestChildSpan( events, - "flue.toolCurrentProbe", - lookupToolSpan?.span.id, + "flue.workflowCurrentProbe", + workflow?.span.id, ); expect(appSpan).toBeDefined(); - expect(appSpan?.span.parentIds).toEqual([lookupToolSpan?.span.id]); - expect(appSpan?.output).toBe("lookup-active"); - expect(workflow).toBeDefined(); - }); - } + expect(appSpan?.span.parentIds).toEqual([workflow?.span.id]); + expect(appSpan?.output).toBe("active"); + }, + ); + + test("makes Flue tool spans current for app spans", testConfig, () => { + const promptSpan = findFlueOperation(events, "flue.prompt"); + const lookupToolSpan = findMatchingDescendants( + events, + promptSpan, + (event) => event.span.name === "tool:lookup", + )[0]; + const appSpan = findLatestChildSpan( + events, + "flue.toolCurrentProbe", + lookupToolSpan?.span.id, + ); - test("captures observe-based Flue operation spans", testConfig, () => { + expect(appSpan).toBeDefined(); + expect(appSpan?.span.parentIds).toEqual([lookupToolSpan?.span.id]); + expect(appSpan?.output).toBe("lookup-active"); + }); + + test("captures Flue operation spans", testConfig, () => { for (const flueSpanName of [ "flue.prompt", "flue.skill", @@ -334,10 +296,10 @@ export function defineFlueInstrumentationAssertions(options: { testConfig, () => { const promptSpan = findFlueOperation(events, "flue.prompt"); - const promptChildren = findFlueDescendants( + const promptChildren = findMatchingDescendants( events, promptSpan, - (event) => isFlueChildSpan(event, expectAmbientContext), + isFlueChildSpan, ); const promptTurns = promptChildren.filter( (event) => event.span.name === "flue.turn", @@ -349,7 +311,7 @@ export function defineFlueInstrumentationAssertions(options: { const compactSpan = findFlueOperation(events, "flue.compact"); const allLlmSpans = [promptSpan, skillSpan, compactSpan].flatMap( (span) => - findFlueDescendants( + findMatchingDescendants( events, span, (event) => event.span.name === "flue.turn", @@ -359,12 +321,12 @@ export function defineFlueInstrumentationAssertions(options: { (event) => event.span.name === "tool:lookup", ); const taskSpan = findFlueOperation(events, "flue.task"); - const nestedTaskSpans = findFlueDescendants( + const nestedTaskSpans = findMatchingDescendants( events, taskSpan, (event) => event.span.name === "flue.task", ); - const compaction = findFlueDescendants( + const compaction = findMatchingDescendants( events, compactSpan, (event) => event.span.name?.startsWith("compaction:") === true, @@ -414,15 +376,11 @@ export function defineFlueInstrumentationAssertions(options: { }); test("matches the span tree snapshot", testConfig, async () => { - await matchSpanTreeSnapshot( - buildSpanTree(events, expectAmbientContext), - snapshotPath, - { - normalize: { - additionalProviderIdKeys: ["messageId"], - }, + await matchSpanTreeSnapshot(buildSpanTree(events), snapshotPath, { + normalize: { + additionalProviderIdKeys: ["messageId"], }, - ); + }); }); }); } diff --git a/e2e/scenarios/flue-instrumentation/package.json b/e2e/scenarios/flue-instrumentation/package.json index 456dcc68f..5a98aeb76 100644 --- a/e2e/scenarios/flue-instrumentation/package.json +++ b/e2e/scenarios/flue-instrumentation/package.json @@ -5,19 +5,11 @@ "braintrustScenario": { "bump": { "dependencies": { - "flue-cli-v0-8-latest": { - "package": "@flue/cli", - "range": ">=0.8.0 <0.9.0" - }, "flue-cli-v1-latest": { "package": "@flue/cli", "range": ">=1.0.0-beta.3 <2.0.0-0", "allowPrerelease": true }, - "flue-runtime-v0-8-latest": { - "package": "@flue/runtime", - "range": ">=0.8.0 <0.9.0" - }, "flue-runtime-v1-latest": { "package": "@flue/runtime", "range": ">=1.0.0-beta.3 <2.0.0-0", @@ -31,13 +23,9 @@ } }, "dependencies": { - "@flue/cli": "0.8.0", - "@flue/runtime": "0.8.0", - "flue-cli-v0-8-latest": "npm:@flue/cli@0.8.1", - "flue-cli-v1": "npm:@flue/cli@1.0.0-beta.3", + "@flue/cli": "1.0.0-beta.3", + "@flue/runtime": "1.0.0-beta.3", "flue-cli-v1-latest": "npm:@flue/cli@1.0.0-beta.9", - "flue-runtime-v0-8-latest": "npm:@flue/runtime@0.8.1", - "flue-runtime-v1": "npm:@flue/runtime@1.0.0-beta.3", "flue-runtime-v1-latest": "npm:@flue/runtime@1.0.0-beta.9", "flue-runtime-v2": "npm:@flue/runtime@2.0.0", "flue-runtime-v2-latest": "npm:@flue/runtime@2.0.1", @@ -46,10 +34,6 @@ }, "pnpm": { "overrides": { - "@flue/runtime@0.8.0>@earendil-works/pi-agent-core": "0.75.4", - "@flue/runtime@0.8.0>@earendil-works/pi-ai": "0.75.4", - "@flue/runtime@0.8.1>@earendil-works/pi-agent-core": "0.75.4", - "@flue/runtime@0.8.1>@earendil-works/pi-ai": "0.75.4", "@hono/node-server@>=2.0.0 <2.0.10": "^2.0.10", "fast-uri@<3.1.5": "^3.1.5", "hono@>=4.0.0 <4.12.27": "^4.12.27", diff --git a/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml b/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml index d1b7e728c..d1eec91bd 100644 --- a/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/flue-instrumentation/pnpm-lock.yaml @@ -5,10 +5,6 @@ settings: excludeLinksFromLockfile: false overrides: - '@flue/runtime@0.8.0>@earendil-works/pi-agent-core': 0.75.4 - '@flue/runtime@0.8.0>@earendil-works/pi-ai': 0.75.4 - '@flue/runtime@0.8.1>@earendil-works/pi-agent-core': 0.75.4 - '@flue/runtime@0.8.1>@earendil-works/pi-ai': 0.75.4 '@hono/node-server@>=2.0.0 <2.0.10': ^2.0.10 fast-uri@<3.1.5: ^3.1.5 hono@>=4.0.0 <4.12.27: ^4.12.27 @@ -23,38 +19,26 @@ importers: .: dependencies: '@flue/cli': - specifier: 0.8.0 - version: 0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) + specifier: 1.0.0-beta.3 + version: 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) '@flue/runtime': - specifier: 0.8.0 - version: 0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - flue-cli-v0-8-latest: - specifier: npm:@flue/cli@0.8.1 - version: '@flue/cli@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' - flue-cli-v1: - specifier: npm:@flue/cli@1.0.0-beta.3 - version: '@flue/cli@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' + specifier: 1.0.0-beta.3 + version: 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) flue-cli-v1-latest: specifier: npm:@flue/cli@1.0.0-beta.9 - version: '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' - flue-runtime-v0-8-latest: - specifier: npm:@flue/runtime@0.8.1 - version: '@flue/runtime@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' - flue-runtime-v1: - specifier: npm:@flue/runtime@1.0.0-beta.3 - version: '@flue/runtime@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' + version: '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' flue-runtime-v1-latest: specifier: npm:@flue/runtime@1.0.0-beta.9 - version: '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' + version: '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)' flue-runtime-v2: specifier: npm:@flue/runtime@2.0.0 - version: '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)' + version: '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)' flue-runtime-v2-latest: specifier: npm:@flue/runtime@2.0.1 - version: '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)' + version: '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)' pi-ai-v2: specifier: npm:@earendil-works/pi-ai@0.83.0 - version: '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)' + version: '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)' valibot: specifier: 1.4.2 version: 1.4.2 @@ -191,12 +175,6 @@ packages: workerd: optional: true - '@cloudflare/vite-plugin@1.39.0': - resolution: {integrity: sha512-AHC+KSR+3dtGu7Ab7I0Ode4Whx12TxMEmiZt7w+Fc3/2wYNByIzbb6cndWZ78tnveFdO1xhNLv1YaNngxGtOPg==} - peerDependencies: - vite: ^6.1.0 || ^7.0.0 || ^8.0.0 - wrangler: ^4.95.0 - '@cloudflare/vite-plugin@1.50.0': resolution: {integrity: sha512-zIhZim7Kr7OC22rlOkU81BLj0oRlUOqPRX+E6RU3hyRYg4xU1MbA3yiMIwFONr+jc/uV7Fxs20NlXrB89Fqjqg==} hasBin: true @@ -273,10 +251,6 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - '@earendil-works/pi-agent-core@0.75.4': - resolution: {integrity: sha512-cGYbysb4EqUf0B28OeqFq2ppm1XF3bYBOP71q9dv38yf/UJfzMjiXBeNelrcio+QWIoVrW+xzYm7sMzYIUc9Og==} - engines: {node: '>=22.19.0'} - '@earendil-works/pi-agent-core@0.79.10': resolution: {integrity: sha512-XKxgdjhcPuyjrthCOFSgfzT3xZ1uBrJ1IMVDxci1to6hIN6BIg9J5iY8q0pGXK1DLgATLP23da+1UyZLwA360Q==} engines: {node: '>=22.19.0'} @@ -289,11 +263,6 @@ packages: resolution: {integrity: sha512-RorGp9OH5l3ElpuC5a5ZQ2eWcchZGXflXRzVGkV99y3y6tT+LLNyxoYIdVKvTKWEObwhExeQbTH0fI2tE4iX4g==} engines: {node: '>=22.19.0'} - '@earendil-works/pi-ai@0.75.4': - resolution: {integrity: sha512-m/w8Hh3vQ0rAycwJiJWdzkypkn4295f4eq/966lDRy8aX5sk6bgYXH8TQmL16TO7Uwc7MbJG0QoyFHgX8RqXUQ==} - engines: {node: '>=22.19.0'} - hasBin: true - '@earendil-works/pi-ai@0.79.10': resolution: {integrity: sha512-9jR23tOl0BIUdQMn70Gr72xYBpM7Xgl9Lyv7gAnU1USfkNRuYG/f/edLl+n/Dp/RafDW3JI4DF7y/GhgkORuew==} engines: {node: '>=22.19.0'} @@ -477,26 +446,6 @@ packages: cpu: [x64] os: [win32] - '@flue/cli@0.8.0': - resolution: {integrity: sha512-iWTVoSCZ0/D5T5O4E+dha9IT7yx1/qqjsEGLtedd/wnlp2nK45qTrjutBS2pgenGNzdQ6wlPzMPkP8MA9z49yw==} - engines: {node: '>=22.18.0'} - hasBin: true - peerDependencies: - wrangler: ^4.94.0 - peerDependenciesMeta: - wrangler: - optional: true - - '@flue/cli@0.8.1': - resolution: {integrity: sha512-8aeaSf7RsXj4A4P+eQgTj8l2+17NScrVCK3UcbMuQ2pwAu84v5t6SQR2XvYuqActlidTIFR6+xxWS7dR9T7yZw==} - engines: {node: '>=22.18.0'} - hasBin: true - peerDependencies: - wrangler: ^4.94.0 - peerDependenciesMeta: - wrangler: - optional: true - '@flue/cli@1.0.0-beta.3': resolution: {integrity: sha512-i8Lo0koB0gcy3bbCp+kwA1C522PsxvN8ydexDwR3rMkWisitG3FVmqmsLPkWBetspmXBLDbyDq39+AdSoXf4EQ==} engines: {node: '>=22.19.0'} @@ -507,14 +456,6 @@ packages: engines: {node: '>=22.19.0'} hasBin: true - '@flue/runtime@0.8.0': - resolution: {integrity: sha512-GCXJOZtb6s+HuMBzB4I41uQzxX7pELfdorD3gnWnkYoNFjHW8D/ncjXMTsczyl2tkP1IPK0C1Ffw0zwGqkmO7g==} - engines: {node: '>=22.18.0'} - - '@flue/runtime@0.8.1': - resolution: {integrity: sha512-nhIiNLr4NmsK6xgYgFt+mTFTKhHxMn++4gjzygYQIUqQK0GR4hgIjpvosl/361Xx087+8N0wNWh2MEVStoZCIg==} - engines: {node: '>=22.18.0'} - '@flue/runtime@1.0.0-beta.3': resolution: {integrity: sha512-VkWz6lRaS73fu6Y8Wh+CjAHjrLmICUmDwURQCA7dtXWP2O99UyvDQ/5Cbg9DYpsYZpaFuOhWzOCj+Igc4F/LbA==} engines: {node: '>=22.19.0'} @@ -903,9 +844,6 @@ packages: '@microsoft/fetch-event-source@2.0.1': resolution: {integrity: sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==} - '@mistralai/mistralai@2.2.1': - resolution: {integrity: sha512-uKU8CZmL2RzYKmplsU01hii4p3pe4HqJefpWNRWXm1Tcm0Sm4xXfwSLIy4k7ZCPlbETCGcp69E7hZs+WOJ5itQ==} - '@mistralai/mistralai@2.2.6': resolution: {integrity: sha512-W8pX7zHxjJvMIpw8JMxeJEleapXX0Q9NPszdNzqkM3MIEoIGPObdodujj+WHteXEvGfaP/AMwlNyRfEzSY6dQQ==} peerDependencies: @@ -2002,11 +1940,6 @@ packages: resolution: {integrity: sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg==} hasBin: true - semver@7.8.0: - resolution: {integrity: sha512-AcM7dV/5ul4EekoQ29Agm5vri8JNqRyj39o0qpX6vDF2GZrtutZl5RwgD1XnZjiTAfncsJhMI48QQH3sN87YNA==} - engines: {node: '>=10'} - hasBin: true - semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} @@ -2154,10 +2087,6 @@ packages: resolution: {integrity: sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==} engines: {node: '>=20.18.1'} - undici@7.28.0: - resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} - engines: {node: '>=20.18.1'} - unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} @@ -2553,19 +2482,6 @@ snapshots: optionalDependencies: workerd: 1.20260730.1 - '@cloudflare/vite-plugin@1.39.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(workerd@1.20260730.1)(wrangler@4.95.0)': - dependencies: - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260730.1) - miniflare: 4.20260526.0 - unenv: 2.0.0-rc.24 - vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0) - wrangler: 4.95.0 - ws: 8.20.1 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - workerd - '@cloudflare/vite-plugin@1.50.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(wrangler@4.95.0)': dependencies: '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260730.1) @@ -2618,9 +2534,9 @@ snapshots: '@microsoft/fetch-event-source': 2.0.1 fastq: 1.20.1 - '@earendil-works/pi-agent-core@0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) ignore: 7.0.5 typebox: 1.1.38 yaml: 2.9.0 @@ -2632,9 +2548,9 @@ snapshots: - ws - zod - '@earendil-works/pi-agent-core@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) ignore: 7.0.5 typebox: 1.1.38 yaml: 2.9.0 @@ -2646,23 +2562,9 @@ snapshots: - ws - zod - '@earendil-works/pi-agent-core@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-agent-core@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - ignore: 7.0.5 - typebox: 1.1.38 - yaml: 2.9.0 - transitivePeerDependencies: - - '@modelcontextprotocol/sdk' - - bufferutil - - supports-color - - utf-8-validate - - ws - - zod - - '@earendil-works/pi-agent-core@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': - dependencies: - '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) diff: 8.0.4 ignore: 7.0.5 typebox: 1.3.7 @@ -2675,26 +2577,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': - dependencies: - '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) - '@aws-sdk/client-bedrock-runtime': 3.1048.0 - '@google/genai': 1.52.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3)) - '@mistralai/mistralai': 2.2.1 - http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) - partial-json: 0.1.7 - typebox: 1.1.38 - transitivePeerDependencies: - - '@modelcontextprotocol/sdk' - - bufferutil - - supports-color - - utf-8-validate - - ws - - zod - - '@earendil-works/pi-ai@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -2704,7 +2587,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.1.38 transitivePeerDependencies: @@ -2715,7 +2598,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -2725,7 +2608,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.1.38 transitivePeerDependencies: @@ -2736,7 +2619,7 @@ snapshots: - ws - zod - '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@earendil-works/pi-ai@0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: '@anthropic-ai/sdk': 0.91.1(zod@4.4.3) '@aws-sdk/client-bedrock-runtime': 3.1048.0 @@ -2746,7 +2629,7 @@ snapshots: '@smithy/node-http-handler': 4.7.3 http-proxy-agent: 7.0.2 https-proxy-agent: 7.0.6 - openai: 6.26.0(ws@8.20.1)(zod@4.4.3) + openai: 6.26.0(ws@8.21.0)(zod@4.4.3) partial-json: 0.1.7 typebox: 1.3.7 transitivePeerDependencies: @@ -2856,88 +2739,10 @@ snapshots: '@esbuild/win32-x64@0.27.3': optional: true - '@flue/cli@0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@cloudflare/vite-plugin': 1.39.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(workerd@1.20260730.1)(wrangler@4.95.0) - '@flue/runtime': 0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@vercel/detect-agent': 1.2.3 - package-up: 5.0.0 - valibot: 1.4.2 - vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0) - optionalDependencies: - wrangler: 4.95.0 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - '@types/node' - - '@vitejs/devtools' - - arktype - - bufferutil - - effect - - esbuild - - jiti - - less - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - sury - - terser - - tsx - - typebox - - typescript - - utf-8-validate - - workerd - - yaml - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/cli@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(typebox@1.3.7)(workerd@1.20260730.1)(wrangler@4.95.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@cloudflare/vite-plugin': 1.39.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(workerd@1.20260730.1)(wrangler@4.95.0) - '@flue/runtime': 0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@vercel/detect-agent': 1.2.3 - package-up: 5.0.0 - valibot: 1.4.2 - vite: 8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0) - optionalDependencies: - wrangler: 4.95.0 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - '@types/node' - - '@vitejs/devtools' - - arktype - - bufferutil - - effect - - esbuild - - jiti - - less - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - sury - - terser - - tsx - - typebox - - typescript - - utf-8-validate - - workerd - - yaml - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/cli@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/cli@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: '@cloudflare/vite-plugin': 1.50.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(wrangler@4.95.0) - '@flue/runtime': 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) + '@flue/runtime': 1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) '@flue/sdk': 1.0.0-beta.3 '@hono/node-server': 2.0.11(hono@4.12.31) '@vercel/detect-agent': 1.2.3 @@ -2979,10 +2784,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.20.1)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/cli@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@types/node@25.9.1)(esbuild@0.27.3)(hono@4.12.31)(typebox@1.3.7)(wrangler@4.95.0)(ws@8.21.0)(yaml@2.9.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: '@cloudflare/vite-plugin': 1.50.0(vite@8.0.14(@types/node@25.9.1)(esbuild@0.27.3)(yaml@2.9.0))(wrangler@4.95.0) - '@flue/runtime': 1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) + '@flue/runtime': 1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) '@flue/sdk': 1.0.0-beta.9 '@hono/node-server': 2.0.11(hono@4.12.31) '@vercel/detect-agent': 1.2.3 @@ -3024,80 +2829,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/runtime@0.8.0(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@earendil-works/pi-agent-core': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@hono/node-server': 2.0.11(hono@4.12.31) - '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - '@standard-community/standard-json': 0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@standard-community/standard-openapi': 0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3) - '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) - hono: 4.12.31 - hono-openapi: 1.3.0(@hono/standard-validator@0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31))(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-community/standard-openapi@0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3))(@types/json-schema@7.0.15)(hono@4.12.31)(openapi-types@12.1.3) - js-yaml: 4.1.1 - just-bash: 3.0.1 - openapi-types: 12.1.3 - quansync: 0.2.11 - ulidx: 2.4.1 - valibot: 1.4.2 - ws: 8.20.1 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - arktype - - bufferutil - - effect - - supports-color - - sury - - typebox - - typescript - - utf-8-validate - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/runtime@0.8.1(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': - dependencies: - '@earendil-works/pi-agent-core': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.75.4(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@hono/node-server': 2.0.11(hono@4.12.31) - '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) - '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) - '@standard-community/standard-json': 0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3) - '@standard-community/standard-openapi': 0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3) - '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) - hono: 4.12.31 - hono-openapi: 1.3.0(@hono/standard-validator@0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31))(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-community/standard-openapi@0.2.9(@standard-community/standard-json@0.3.5(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(@valibot/to-json-schema@1.7.0(valibot@1.4.2))(quansync@0.2.11)(typebox@1.3.7)(valibot@1.4.2)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3))(@standard-schema/spec@1.1.0)(openapi-types@12.1.3)(typebox@1.3.7)(valibot@1.4.2)(zod@4.4.3))(@types/json-schema@7.0.15)(hono@4.12.31)(openapi-types@12.1.3) - js-yaml: 4.1.1 - just-bash: 3.0.1 - openapi-types: 12.1.3 - quansync: 0.2.11 - ulidx: 2.4.1 - valibot: 1.4.2 - ws: 8.20.1 - transitivePeerDependencies: - - '@cfworker/json-schema' - - '@standard-schema/spec' - - '@types/json-schema' - - arktype - - bufferutil - - effect - - supports-color - - sury - - typebox - - typescript - - utf-8-validate - - zod - - zod-openapi - - zod-to-json-schema - - '@flue/runtime@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/runtime@1.0.0-beta.3(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.79.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) @@ -3129,10 +2864,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.20.1)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': + '@flue/runtime@1.0.0-beta.9(@standard-schema/spec@1.1.0)(@types/json-schema@7.0.15)(typebox@1.3.7)(ws@8.21.0)(zod-to-json-schema@3.25.2(zod@4.4.3))(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.80.10(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@hono/standard-validator': 0.2.2(@standard-schema/spec@1.1.0)(hono@4.12.31) '@modelcontextprotocol/sdk': 1.29.0(zod@4.4.3) @@ -3164,10 +2899,10 @@ snapshots: - zod-openapi - zod-to-json-schema - '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@flue/runtime@2.0.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@modelcontextprotocol/client': 2.0.0 '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) @@ -3184,10 +2919,10 @@ snapshots: - ws - zod - '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3)': + '@flue/runtime@2.0.1(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3)': dependencies: - '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) - '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.20.1)(zod@4.4.3) + '@earendil-works/pi-agent-core': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) + '@earendil-works/pi-ai': 0.83.0(@modelcontextprotocol/sdk@1.29.0(zod@4.4.3))(ws@8.21.0)(zod@4.4.3) '@hono/node-server': 2.0.11(hono@4.12.31) '@modelcontextprotocol/client': 2.0.0 '@valibot/to-json-schema': 1.7.0(valibot@1.4.2) @@ -3407,7 +3142,7 @@ snapshots: '@img/sharp-wasm32@0.34.5': dependencies: - '@emnapi/runtime': 1.10.0 + '@emnapi/runtime': 1.11.3 optional: true '@img/sharp-wasm32@0.35.2': @@ -3467,15 +3202,6 @@ snapshots: '@microsoft/fetch-event-source@2.0.1': {} - '@mistralai/mistralai@2.2.1': - dependencies: - ws: 8.20.1 - zod: 4.4.3 - zod-to-json-schema: 3.25.2(zod@4.4.3) - transitivePeerDependencies: - - bufferutil - - utf-8-validate - '@mistralai/mistralai@2.2.6(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/semantic-conventions': 1.43.0 @@ -4303,7 +4029,7 @@ snapshots: dependencies: '@cspotcode/source-map-support': 0.8.1 sharp: 0.35.2 - undici: 7.28.0 + undici: 7.29.0 workerd: 1.20260730.1 ws: 8.21.0 youch: 4.1.0-beta.10 @@ -4336,7 +4062,7 @@ snapshots: node-abi@3.92.0: dependencies: - semver: 7.8.0 + semver: 7.8.5 optional: true node-addon-api@8.7.0: @@ -4371,9 +4097,9 @@ snapshots: dependencies: wrappy: 1.0.2 - openai@6.26.0(ws@8.20.1)(zod@4.4.3): + openai@6.26.0(ws@8.21.0)(zod@4.4.3): optionalDependencies: - ws: 8.20.1 + ws: 8.21.0 zod: 4.4.3 openapi-types@12.1.3: {} @@ -4561,8 +4287,6 @@ snapshots: dependencies: commander: 6.2.1 - semver@7.8.0: {} - semver@7.8.5: {} send@1.2.1: @@ -4596,7 +4320,7 @@ snapshots: dependencies: '@img/colour': 1.1.0 detect-libc: 2.1.2 - semver: 7.8.0 + semver: 7.8.5 optionalDependencies: '@img/sharp-darwin-arm64': 0.34.5 '@img/sharp-darwin-x64': 0.34.5 @@ -4788,8 +4512,6 @@ snapshots: undici@7.29.0: {} - undici@7.28.0: {} - unenv@2.0.0-rc.24: dependencies: pathe: 2.0.3 diff --git a/e2e/scenarios/flue-instrumentation/scenario.cli.mjs b/e2e/scenarios/flue-instrumentation/scenario.cli.mjs deleted file mode 100644 index bbbb961fd..000000000 --- a/e2e/scenarios/flue-instrumentation/scenario.cli.mjs +++ /dev/null @@ -1,4 +0,0 @@ -const { runCliFlueInstrumentation, runMain } = - await import("./scenario.impl.mjs"); - -runMain(runCliFlueInstrumentation); diff --git a/e2e/scenarios/flue-instrumentation/scenario.impl.mjs b/e2e/scenarios/flue-instrumentation/scenario.impl.mjs index e0c4a5ce2..ba30b1543 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.impl.mjs +++ b/e2e/scenarios/flue-instrumentation/scenario.impl.mjs @@ -1,5 +1,4 @@ import { spawn } from "node:child_process"; -import { access, rm } from "node:fs/promises"; import { createServer } from "node:net"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -15,7 +14,6 @@ const flueCliPath = path.join( "bin", "flue.mjs", ); -const braintrustHookNodeOption = "--import=braintrust/hook.mjs"; function workflowPayload() { return { @@ -27,27 +25,23 @@ function workflowPayload() { }; } -export async function runNodeFlueInstrumentationScenario(options) { - const env = scenarioEnv(options); - const outputDir = path.join(process.cwd(), ".flue-build", options.outputName); - await runFlueCli( - [ - "build", - "--target", - "node", - "--root", - process.cwd(), - "--output", - outputDir, - ], - env, - ); +export async function runFlueInstrumentationScenario() { + const outputDir = path.join(process.cwd(), ".flue-build", "explicit"); + await runFlueCli([ + "build", + "--target", + "node", + "--root", + process.cwd(), + "--output", + outputDir, + ]); const port = await getFreePort(); const child = spawn(process.execPath, [path.join(outputDir, "server.mjs")], { cwd: process.cwd(), env: { - ...env, + ...process.env, PORT: String(port), }, stdio: ["ignore", "pipe", "pipe"], @@ -96,69 +90,10 @@ export async function runNodeFlueInstrumentationScenario(options) { } } -export async function runCliFlueInstrumentationScenario() { - const flushFile = path.join(process.cwd(), ".flue-build", "cli-flushed"); - await rm(flushFile, { force: true }); - await runFlueCli( - [ - "run", - "instrumentation", - "--target", - "node", - process.env.FLUE_E2E_INPUT_FLAG ?? "--payload", - JSON.stringify(workflowPayload()), - "--root", - process.cwd(), - ], - { - ...scenarioEnv({ autoHook: true, explicitObserve: false }), - FLUE_E2E_FLUSH_FILE: flushFile, - }, - ); - await waitForFile(flushFile); -} - -export function runExplicitFlueInstrumentation() { - return runNodeFlueInstrumentationScenario({ - autoHook: false, - explicitObserve: true, - outputName: "explicit", - }); -} - -export function runAutoFlueInstrumentation() { - return runNodeFlueInstrumentationScenario({ - autoHook: true, - explicitObserve: false, - outputName: "auto-hook", - }); -} - -export function runCliFlueInstrumentation() { - return runCliFlueInstrumentationScenario(); -} - -function scenarioEnv({ autoHook, explicitObserve }) { - const env = { - ...process.env, - FLUE_E2E_EXPLICIT_OBSERVE: explicitObserve ? "1" : "0", - }; - if (!autoHook) { - return env; - } - return { - ...env, - NODE_OPTIONS: [env.NODE_OPTIONS, braintrustHookNodeOption] - .filter(Boolean) - .join(" "), - }; -} - -async function runFlueCli(args, env) { +async function runFlueCli(args) { await new Promise((resolve, reject) => { const child = spawn(process.execPath, [flueCliPath, ...args], { cwd: process.cwd(), - env, stdio: ["ignore", "pipe", "pipe"], }); @@ -245,27 +180,3 @@ async function stopChild(child) { }, 5_000).unref(); }); } - -async function waitForFile(filePath) { - const startedAt = Date.now(); - while (true) { - try { - await access(filePath); - return; - } catch { - if (Date.now() - startedAt > 30_000) { - throw new Error( - `timed out waiting for Flue e2e flush marker: ${filePath}`, - ); - } - await new Promise((resolve) => setTimeout(resolve, 50)); - } - } -} - -export function runMain(main) { - void main().catch((error) => { - console.error(error); - process.exitCode = 1; - }); -} diff --git a/e2e/scenarios/flue-instrumentation/scenario.mjs b/e2e/scenarios/flue-instrumentation/scenario.mjs deleted file mode 100644 index a9bd9d94e..000000000 --- a/e2e/scenarios/flue-instrumentation/scenario.mjs +++ /dev/null @@ -1,8 +0,0 @@ -import "braintrust"; - -await new Promise((resolve) => setImmediate(resolve)); - -const { runAutoFlueInstrumentation, runMain } = - await import("./scenario.impl.mjs"); - -runMain(runAutoFlueInstrumentation); diff --git a/e2e/scenarios/flue-instrumentation/scenario.test.ts b/e2e/scenarios/flue-instrumentation/scenario.test.ts index 73a27bbf7..22d9941be 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.test.ts +++ b/e2e/scenarios/flue-instrumentation/scenario.test.ts @@ -10,49 +10,19 @@ import { defineFlueInstrumentationAssertions } from "./assertions"; import { defineFlueV2InstrumentationAssertions } from "./v2-assertions"; const originalScenarioDir = resolveScenarioDir(import.meta.url); -const generatedScenarioRoot = path.resolve( - originalScenarioDir, - "../../.bt-tmp/generated-scenarios/flue-instrumentation", -); const TIMEOUT_MS = 120_000; -const flueV1ScenarioSourceDir = await prepareGeneratedFlueV1ScenarioSource(); const flueScenarios = await Promise.all([ prepareFlueScenario({ cliPackageName: "@flue/cli", - expectAmbientContext: false, - label: "v0.8 pinned", - runtimePackageName: "@flue/runtime", - sourceDir: originalScenarioDir, - supportsAutoInstrumentation: true, - variantKey: "flue-v0-8-0", - }), - prepareFlueScenario({ - cliPackageName: "flue-cli-v0-8-latest", - expectAmbientContext: false, - label: "v0.8 latest", - modelName: "openai/gpt-5.4-nano", - runtimePackageName: "flue-runtime-v0-8-latest", - sourceDir: originalScenarioDir, - supportsAutoInstrumentation: true, - variantKey: "flue-v0-8-latest", - }), - prepareFlueScenario({ - cliPackageName: "flue-cli-v1", - expectAmbientContext: true, label: "v1 pinned", - runtimePackageName: "flue-runtime-v1", - sourceDir: flueV1ScenarioSourceDir, - supportsAutoInstrumentation: false, + runtimePackageName: "@flue/runtime", variantKey: "flue-v1-0-0-beta-3", }), prepareFlueScenario({ cliPackageName: "flue-cli-v1-latest", - expectAmbientContext: true, label: "v1 latest", modelName: "openai/gpt-5.4-nano", runtimePackageName: "flue-runtime-v1-latest", - sourceDir: flueV1ScenarioSourceDir, - supportsAutoInstrumentation: false, variantKey: "flue-v1-latest", }), ]); @@ -84,7 +54,6 @@ describe.sequential("flue variants", () => { for (const scenario of flueScenarios) { describe.sequential(`flue ${scenario.label} (${scenario.version})`, () => { defineFlueInstrumentationAssertions({ - expectAmbientContext: scenario.expectAmbientContext, name: "explicit instrumentation", runScenario: async ({ runScenarioDir }) => { await runScenarioDir({ @@ -102,54 +71,6 @@ describe.sequential("flue variants", () => { testFileUrl: import.meta.url, timeoutMs: TIMEOUT_MS, }); - - if (scenario.supportsAutoInstrumentation) { - defineFlueInstrumentationAssertions({ - expectAmbientContext: scenario.expectAmbientContext, - name: "auto-hook instrumentation", - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: "scenario.mjs", - env: scenario.env, - nodeArgs: ["--import", "braintrust/hook.mjs"], - runContext: { - originalScenarioDir, - variantKey: scenario.variantKey, - }, - scenarioDir: scenario.scenarioDir, - timeoutMs: TIMEOUT_MS, - }); - }, - snapshotName: `${scenario.variantKey}-auto-hook`, - testFileUrl: import.meta.url, - timeoutMs: TIMEOUT_MS, - }); - - defineFlueInstrumentationAssertions({ - expectAmbientContext: scenario.expectAmbientContext, - name: "cli instrumentation", - runScenario: async ({ runNodeScenarioDir }) => { - await runNodeScenarioDir({ - entry: "scenario.cli.mjs", - env: { - ...scenario.env, - ...(scenario.inputFlag - ? { FLUE_E2E_INPUT_FLAG: scenario.inputFlag } - : {}), - }, - runContext: { - originalScenarioDir, - variantKey: scenario.variantKey, - }, - scenarioDir: scenario.scenarioDir, - timeoutMs: TIMEOUT_MS, - }); - }, - snapshotName: `${scenario.variantKey}-cli`, - testFileUrl: import.meta.url, - timeoutMs: TIMEOUT_MS, - }); - } }); } @@ -169,17 +90,13 @@ describe.sequential("flue variants", () => { async function prepareFlueScenario(options: { cliPackageName: string; - expectAmbientContext: boolean; - inputFlag?: string; label: string; modelName?: string; runtimePackageName: string; - sourceDir: string; - supportsAutoInstrumentation: boolean; variantKey: string; }) { const scenarioDir = await prepareScenarioDir({ - scenarioDir: options.sourceDir, + scenarioDir: originalScenarioDir, }); const [cliPackageDir, runtimePackageDir] = await Promise.all([ fs.realpath(path.join(scenarioDir, "node_modules", options.cliPackageName)), @@ -203,7 +120,6 @@ async function prepareFlueScenario(options: { ), ]); return { - ...options, env: { FLUE_CLI_PACKAGE_NAME: options.cliPackageName, FLUE_RUNTIME_PACKAGE_NAME: options.runtimePackageName, @@ -216,39 +132,12 @@ async function prepareFlueScenario(options: { } : {}), }, + label: options.label, scenarioDir, + variantKey: options.variantKey, version: await readInstalledPackageVersion( scenarioDir, options.runtimePackageName, ), }; } - -async function prepareGeneratedFlueV1ScenarioSource() { - const sourceDir = path.join(generatedScenarioRoot, "flue-v1"); - await fs.rm(sourceDir, { force: true, recursive: true }); - await fs.mkdir(sourceDir, { recursive: true }); - await fs.cp(originalScenarioDir, sourceDir, { - filter(source) { - const relative = path.relative(originalScenarioDir, source); - return ( - relative === "" || - !["__cassettes__", "__snapshots__", "node_modules", "versions"].some( - (name) => - relative === name || relative.startsWith(`${name}${path.sep}`), - ) - ); - }, - recursive: true, - }); - await fs.cp(path.join(originalScenarioDir, "versions", "v1"), sourceDir, { - filter(source) { - return ( - source !== - path.join(originalScenarioDir, "versions", "v1", "package.json") - ); - }, - recursive: true, - }); - return sourceDir; -} diff --git a/e2e/scenarios/flue-instrumentation/scenario.ts b/e2e/scenarios/flue-instrumentation/scenario.ts index b96ed4882..a5e57e15d 100644 --- a/e2e/scenarios/flue-instrumentation/scenario.ts +++ b/e2e/scenarios/flue-instrumentation/scenario.ts @@ -1,3 +1,3 @@ -import { runExplicitFlueInstrumentation, runMain } from "./scenario.impl.mjs"; +import { runFlueInstrumentationScenario } from "./scenario.impl.mjs"; -runMain(runExplicitFlueInstrumentation); +await runFlueInstrumentationScenario(); diff --git a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs b/e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs deleted file mode 100644 index 6680c7ae8..000000000 --- a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/app.mjs +++ /dev/null @@ -1,112 +0,0 @@ -import { flush, initLogger } from "braintrust"; -import { Hono } from "hono"; -import { mkdir, writeFile } from "node:fs/promises"; -import { dirname } from "node:path"; - -const runtimePackageName = - process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "flue-runtime-v1"; -const [{ instrument, registerProvider }, { flue }] = await Promise.all([ - import(runtimePackageName), - import(`${runtimePackageName}/routing`), -]); - -function projectName() { - const configured = process.env.BRAINTRUST_E2E_PROJECT_NAME; - if (configured) { - return configured; - } - const testRunId = process.env.BRAINTRUST_E2E_RUN_ID ?? "local"; - return `e2e-flue-instrumentation-${testRunId.toLowerCase().replace(/[^a-z0-9-]/g, "-")}`; -} - -initLogger({ projectName: projectName() }); - -const exitProcess = process.exit.bind(process); -if (process.env.FLUE_E2E_FLUSH_FILE) { - let isExiting = false; - process.exit = (code) => { - if (isExiting) { - return exitProcess(code); - } - isExiting = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - exitProcess(code); - }); - }; -} - -if (process.env.FLUE_E2E_EXPLICIT_OBSERVE === "1") { - const { braintrustFlueInstrumentation } = await import("braintrust"); - instrument(braintrustFlueInstrumentation()); -} - -const openAIBaseUrl = - process.env.OPENAI_BASE_URL ?? process.env.BRAINTRUST_E2E_MODEL_BASE_URL; -if (openAIBaseUrl) { - registerProvider("openai", { baseUrl: openAIBaseUrl }); -} - -const anthropicBaseUrl = process.env.ANTHROPIC_BASE_URL; -if (anthropicBaseUrl) { - registerProvider("anthropic", { - apiKey: process.env.ANTHROPIC_API_KEY ?? "test-key", - baseUrl: anthropicBaseUrl, - }); -} - -let didScheduleFlush = false; -function scheduleFinalFlush(exitAfterFlush = false) { - if (didScheduleFlush) { - return; - } - didScheduleFlush = true; - const keepAlive = setTimeout(() => {}, 30_000); - void flushBeforeExit() - .catch((error) => { - console.error(error); - }) - .finally(() => { - clearTimeout(keepAlive); - if (exitAfterFlush) { - exitProcess(0); - } - }); -} - -process.on("SIGTERM", () => { - scheduleFinalFlush(true); -}); - -process.on("beforeExit", () => { - scheduleFinalFlush(); -}); - -const app = new Hono(); -app.route("/", flue()); - -async function flushBeforeExit() { - await new Promise((resolve) => setTimeout(resolve, 250)); - await flush(); - if (process.env.FLUE_E2E_FLUSH_FILE) { - await mkdir(dirname(process.env.FLUE_E2E_FLUSH_FILE), { - recursive: true, - }); - await writeFile(process.env.FLUE_E2E_FLUSH_FILE, "ok"); - } -} - -export default { - async fetch(request, env, ctx) { - if (new URL(request.url).pathname === "/__braintrust_flush") { - await flush(); - return new Response("ok"); - } - return app.fetch(request, env, ctx); - }, -}; diff --git a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs b/e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs deleted file mode 100644 index 2c8660430..000000000 --- a/e2e/scenarios/flue-instrumentation/versions/v1/.flue/workflows/instrumentation.mjs +++ /dev/null @@ -1,183 +0,0 @@ -import { traced } from "braintrust"; -import * as v from "valibot"; -import { - FLUE_MODEL, - FLUE_REASONING_MODEL, - SCENARIO_NAME, -} from "../../constants.mjs"; - -const runtimePackageName = - process.env.FLUE_RUNTIME_PACKAGE_NAME ?? "flue-runtime-v1"; -const [{ defineAgent, defineTool, defineWorkflow }, { local }] = - await Promise.all([ - import(runtimePackageName), - import(`${runtimePackageName}/node`), - ]); - -function flueModel() { - return process.env.FLUE_E2E_MODEL ?? FLUE_MODEL; -} - -function flueReasoningModel() { - return process.env.FLUE_E2E_REASONING_MODEL ?? FLUE_REASONING_MODEL; -} - -function fluePromptModel() { - return process.env.FLUE_E2E_PROMPT_MODEL ?? flueReasoningModel(); -} - -function fluePromptThinkingLevel() { - return ( - process.env.FLUE_E2E_PROMPT_THINKING_LEVEL ?? flueReasoningThinkingLevel() - ); -} - -function flueReasoningThinkingLevel() { - return process.env.FLUE_E2E_REASONING_THINKING_LEVEL ?? "medium"; -} - -const flueE2EAgent = defineAgent(() => ({ - compaction: { - keepRecentTokens: 1, - reserveTokens: 64, - }, - cwd: process.cwd(), - instructions: [ - "You are a deterministic Flue instrumentation test agent.", - "Follow user instructions exactly.", - "When asked for a marker, output only that marker and no extra text.", - "When running a local skill file, read it yourself and do not delegate it to a task.", - ].join(" "), - model: flueModel(), - sandbox: local({ cwd: process.cwd() }), - thinkingLevel: "off", -})); - -const lookupTool = defineTool({ - description: - "Return a deterministic lookup result with an id needed by web_search.", - input: v.object({ - query: v.string(), - }), - name: "lookup", - run: async ({ input }) => { - await traced( - async (span) => { - span.log({ output: "lookup-active" }); - }, - { - name: "flue.toolCurrentProbe", - event: { - metadata: { - scenario: SCENARIO_NAME, - }, - }, - }, - ); - - return { - id: "flue-session-2026", - query: input.query, - topic: "session instrumentation", - }; - }, -}); - -const webSearchTool = defineTool({ - description: - "Search a deterministic local web index. Requires the id returned by lookup.", - input: v.object({ - lookupId: v.string(), - query: v.string(), - }), - name: "web_search", - run: async ({ input }) => ({ - lookupId: input.lookupId, - query: input.query, - results: [ - { - title: "Flue reasoning stream instrumentation", - url: "https://example.test/flue/reasoning-streams", - }, - ], - }), -}); - -const summarizeSourceTool = defineTool({ - description: - "Summarize the selected deterministic source after web_search returns a URL.", - input: v.object({ - url: v.string(), - }), - name: "summarize_source", - run: async ({ input }) => ({ - summary: - "Flue emits reasoning, tool execution, and LLM turn events separately.", - url: input.url, - }), -}); - -export async function route(_ctx, next) { - await next(); -} - -export default defineWorkflow({ - agent: flueE2EAgent, - input: v.object({ - scenario: v.optional(v.string()), - metadata: v.optional(v.record(v.string(), v.unknown())), - }), - async run({ harness, input }) { - await traced( - async (span) => { - span.log({ output: "active" }); - }, - { - name: "flue.workflowCurrentProbe", - event: { - metadata: { - scenario: SCENARIO_NAME, - }, - }, - }, - ); - - const session = await harness.session("main"); - const skillSession = await harness.session("skill"); - const taskSession = await harness.session("task"); - - await session.prompt( - [ - "Complete this instrumented research flow.", - "Call exactly one tool per turn and wait for each tool result before choosing the next tool.", - 'Step 1: call lookup with query "flue instrumentation".', - 'Step 2: use the lookup result id as lookupId and call web_search with query "Braintrust Flue reasoning stream instrumentation".', - "Step 3: use the first web_search result url and call summarize_source.", - "After summarize_source returns, reply with exactly PROMPT_DONE and no other text.", - ].join(" "), - { - model: fluePromptModel(), - thinkingLevel: fluePromptThinkingLevel(), - tools: [lookupTool, webSearchTool, summarizeSourceTool], - }, - ); - - await skillSession.skill("e2e-flue-skill", { - args: { marker: "SKILL_DONE" }, - model: flueReasoningModel(), - thinkingLevel: "off", - }); - - await taskSession.task("Reply with exactly TASK_DONE and no other text.", { - model: FLUE_MODEL, - thinkingLevel: "off", - }); - - await session.compact(); - - return { - scenario: input.scenario ?? SCENARIO_NAME, - status: "done", - }; - }, -}); diff --git a/integrations/otel-js/src/otel-compat.test.ts b/integrations/otel-js/src/otel-compat.test.ts index 6564da531..d06d9f843 100644 --- a/integrations/otel-js/src/otel-compat.test.ts +++ b/integrations/otel-js/src/otel-compat.test.ts @@ -12,7 +12,7 @@ import { getContextManager, BRAINTRUST_CURRENT_SPAN_STORE, _exportsForTestingOnly, - runEvaluator, + Eval, } from "braintrust"; import { BasicTracerProvider, @@ -371,7 +371,7 @@ describe("OTEL compatibility mode", () => { expect(trace2Id).not.toBe(trace3Id); }); - test("OTEL spans in experiment Eval() inherit experiment_id parent", async () => { + test("OTEL spans in Eval() inherit Braintrust parent context", async () => { const { tracer, exporter, processor } = setupOtelFixture( "experiment-eval-test", ); @@ -379,12 +379,11 @@ describe("OTEL compatibility mode", () => { // Capture BT span info from inside the task const btSpanInfo: Array<{ traceId: string; spanId: string }> = []; - // Use runEvaluator with null experiment to avoid API calls - const result = await runEvaluator( - null, + // Use the public Eval API without making Braintrust API calls. + const result = await Eval( + "test-eval-project", { - projectName: "test-eval-project", - evalName: "otel-eval-test", + experimentName: "otel-eval-test", data: [{ input: 1 }, { input: 2 }], task: async (input: number) => { // Capture the current BT span info @@ -408,9 +407,10 @@ describe("OTEL compatibility mode", () => { }, scores: [], }, - new NoopProgressReporter(), - [], - undefined, + { + noSendLogs: true, + progress: new NoopProgressReporter(), + }, ); // Verify we captured BT span info diff --git a/js/dev/authorize.ts b/js/dev/authorize.ts deleted file mode 100644 index 4f7592d17..000000000 --- a/js/dev/authorize.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import { IncomingHttpHeaders } from "http"; -import createError from "http-errors"; -import { LRUCache } from "../src/lru-cache"; -import { BraintrustState, LoginOptions, loginToState } from "../src/logger"; - -export interface RequestContext { - appOrigin: string; - token: string | undefined; - state: BraintrustState | undefined; - projectId: string | undefined; -} -declare module "express" { - interface Request { - ctx?: RequestContext; - } -} - -export function authorizeRequest( - req: Request, - res: Response, - next: NextFunction, -) { - try { - const ctx: RequestContext = { - appOrigin: extractAllowedOrigin(req.headers[ORIGIN_HEADER]), - token: undefined, - state: undefined, - projectId: parseHeader(req.headers, PROJECT_ID_HEADER), - }; - - // Extract token and data from request - if ( - req.headers.authorization || - req.headers[BRAINTRUST_AUTH_TOKEN_HEADER] - ) { - const tokenText = parseBraintrustAuthHeader(req.headers); - if (!tokenText) { - return next(createError(400, "Invalid authorization token format")); - } - ctx.token = tokenText.toLowerCase() === "null" ? undefined : tokenText; - } - - req.ctx = ctx; - - next(); // Proceed to next middleware/controller - } catch (e) { - next(e); - } -} - -const loginCache = new LRUCache({ - max: 32, // TODO: Make this configurable -}); - -async function cachedLogin(options: LoginOptions): Promise { - const key = JSON.stringify(options); - const cached = loginCache.get(key); - if (cached) { - return cached; - } - - const state = await loginToState(options); - loginCache.set(key, state); - return state; -} - -export function makeCheckAuthorized(allowedOrgName: string | undefined) { - return async (req: Request, _res: Response, next: NextFunction) => { - if (!req.ctx?.token) { - return next(createError(401, "Unauthorized")); - } - - try { - const orgName = parseHeader(req.headers, "x-bt-org-name"); - - if (!orgName) { - return next(createError(400, "Missing x-bt-org-name header")); - } - - if (allowedOrgName && allowedOrgName !== orgName) { - const errorMessage = `Org '${orgName}' is not allowed. Only org '${allowedOrgName}' is allowed.`; - return next(createError(403, errorMessage)); - } - - const state = await cachedLogin({ - apiKey: req.ctx?.token, - orgName: orgName, - }); - req.ctx.state = state; - next(); - } catch (e) { - console.error("Authorization error:", e); - return next(createError(401, "Unauthorized")); - } - }; -} - -function parseBraintrustAuthHeader( - headers: IncomingHttpHeaders | Record, -): string | undefined { - const tokenString = parseHeader(headers, BRAINTRUST_AUTH_TOKEN_HEADER); - return tokenString ?? parseAuthHeader(headers) ?? undefined; -} - -function parseHeader( - headers: IncomingHttpHeaders | Record, - headerName: string, -): string | undefined { - const token = headers[headerName]; - let tokenString; - if (typeof token === "string") { - tokenString = token; - } else if (Array.isArray(token) && token.length > 0) { - tokenString = token[0]; - } - - return tokenString; -} - -export type StaticOrigin = - | boolean - | string - | RegExp - | Array; - -export function checkOrigin( - requestOrigin: string | undefined, - callback: (err: Error | null, origin?: StaticOrigin) => void, -) { - if (!requestOrigin) { - return callback(null, true); - } - - // the origins can be glob patterns - for (const origin of WHITELISTED_ORIGINS || []) { - if ( - (origin instanceof RegExp && origin.test(requestOrigin)) || - origin === requestOrigin - ) { - return callback(null, requestOrigin); - } - } - - return callback(null, false); -} - -const BRAINTRUST_AUTH_TOKEN_HEADER = "x-bt-auth-token"; -const ORIGIN_HEADER = "origin"; -const PROJECT_ID_HEADER = "x-bt-project-id"; - -function extractAllowedOrigin(originHeader: string | undefined): string { - let allowedOrigin: string = MAIN_ORIGIN; - checkOrigin(originHeader, (err, origin) => { - if (!err && originHeader && origin) { - allowedOrigin = originHeader; - } - }); - return allowedOrigin; -} - -const MAIN_ORIGIN = "https://www.braintrust.dev"; -const WHITELISTED_ORIGINS = [ - MAIN_ORIGIN, - "https://www.braintrustdata.com", - new RegExp("^https://[^/]+\\.preview\\.braintrust\\.dev$"), -] - .concat( - process.env.WHITELISTED_ORIGIN ? [process.env.WHITELISTED_ORIGIN] : [], - ) - .concat( - process.env.BRAINTRUST_APP_URL ? [process.env.BRAINTRUST_APP_URL] : [], - ); - -function parseAuthHeader( - headers: Record, -) { - const authHeader = headers["authorization"]; - let authValue = null; - if (Array.isArray(authHeader)) { - authValue = authHeader[authHeader.length - 1]; - } else { - authValue = authHeader; - } - - if (!authValue) { - return null; - } - - const parts = authValue.split(" "); - if (parts.length !== 2) { - return null; - } - return parts[1]; -} - -export const baseAllowedHeaders = [ - "Content-Type", - "X-Amz-Date", - "Authorization", - "X-Api-Key", - "X-Amz-Security-Token", - "x-bt-auth-token", - "x-bt-parent", - // These are eval-specific - "x-bt-org-name", - "x-bt-project-id", - "x-bt-stream-fmt", - "x-bt-use-cache", - "x-stainless-os", - "x-stainless-lang", - "x-stainless-package-version", - "x-stainless-runtime", - "x-stainless-runtime-version", - "x-stainless-arch", - "x-bt-use-gateway", -]; diff --git a/js/dev/errorHandler.ts b/js/dev/errorHandler.ts deleted file mode 100644 index 12cd15e24..000000000 --- a/js/dev/errorHandler.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { z } from "zod/v3"; -import { Request, Response, ErrorRequestHandler, NextFunction } from "express"; -import { HttpError } from "http-errors"; - -export const errorHandler: ErrorRequestHandler = ( - err: Error | HttpError, - req: Request, - res: Response, - next: NextFunction, -) => { - if ("status" in err) { - res.status(err.status).json({ - error: { - message: err.message, - status: err.status, - }, - }); - return; - } - if (err instanceof z.ZodError) { - res.status(400).json({ - error: { - message: "Invalid request", - errors: err.errors, - }, - }); - return; - } - - console.error("Internal server error", err); - res.status(500).json({ - error: { - message: "Internal server error", - status: 500, - }, - }); -}; diff --git a/js/dev/index.ts b/js/dev/index.ts deleted file mode 100644 index 7ac9af952..000000000 --- a/js/dev/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { configureNode } from "../src/node/config"; - -configureNode(); - -export { runDevServer } from "./server"; diff --git a/js/dev/server.test.ts b/js/dev/server.test.ts deleted file mode 100644 index 2e1461a7e..000000000 --- a/js/dev/server.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { describe, expect, test } from "vitest"; -import { BraintrustState } from "../src/logger"; -import { _exportsForTestingOnly } from "./server"; -import { evalBodySchema } from "./types"; - -describe("remote eval data", () => { - test("uses a named experiment as the data source", async () => { - const request = evalBodySchema.parse({ - name: "remote-evaluator", - data: { experiment_name: "source-experiment" }, - }); - const state = new BraintrustState({}); - - await expect( - _exportsForTestingOnly.getDataset(state, request.data), - ).resolves.toEqual({ - _type: "BaseExperiment", - name: "source-experiment", - }); - }); -}); diff --git a/js/dev/server.ts b/js/dev/server.ts deleted file mode 100644 index 5db8b467f..000000000 --- a/js/dev/server.ts +++ /dev/null @@ -1,407 +0,0 @@ -import express, { NextFunction, Request, Response } from "express"; -import cors from "cors"; -import { - BaseExperiment, - callEvaluatorData, - Eval, - EvalData, - EvalHooks, - EvalScorer, - EvaluatorDef, - OneOrMoreScores, - classifierName, - scorerName, -} from "../src/framework"; -import { errorHandler } from "./errorHandler"; -import { - authorizeRequest, - baseAllowedHeaders, - makeCheckAuthorized, - checkOrigin, -} from "./authorize"; -import { - type FunctionIdType as FunctionId, - type InvokeFunctionType as InvokeFunctionRequest, - type RunEvalType as RunEvalRequest, - type SSEProgressEventDataType as SSEProgressEventData, -} from "../src/generated_types"; -import { - BaseMetadata, - BraintrustState, - EvalCase, - getSpanParentObject, - initDataset, -} from "../src/logger"; -import { - BT_CURSOR_HEADER, - BT_FOUND_EXISTING_HEADER, - parseParent, -} from "../util/index"; -import { serializeSSEEvent } from "./stream"; -import { - evalBodySchema, - EvaluatorDefinitions, - EvaluatorManifest, - type SerializedParametersContainer, -} from "./types"; -import { EvalParameters, validateParameters } from "../src/eval-parameters"; -import { z } from "zod/v3"; -import { ValidationError } from "ajv"; -import { serializeRemoteEvalParametersContainer } from "../src/framework2"; - -export interface DevServerOpts { - host: string; - port: number; - orgName?: string; -} - -export function runDevServer( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluators: EvaluatorDef[], - opts: DevServerOpts, -) { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const allEvaluators: EvaluatorManifest = Object.fromEntries( - evaluators.map((evaluator) => [evaluator.evalName, evaluator]), - ) as EvaluatorManifest; - - globalThis._lazy_load = false; - - const app = express(); - - app.use(express.json({ limit: "1gb" })); - console.log("Starting server"); - app.use((req, res, next) => { - if (req.headers["access-control-request-private-network"]) { - res.setHeader("Access-Control-Allow-Private-Network", "true"); - } - next(); - }); - - const checkAuthorized = makeCheckAuthorized(opts.orgName); - - app.use( - cors({ - origin: checkOrigin, - methods: ["GET", "PATCH", "POST", "PUT", "DELETE", "OPTIONS"], - allowedHeaders: baseAllowedHeaders, - credentials: true, - exposedHeaders: [ - BT_CURSOR_HEADER, - BT_FOUND_EXISTING_HEADER, - "x-bt-span-id", - "x-bt-span-export", - ], - maxAge: 86400, - }), - ); - - app.use(authorizeRequest); - - app.get("/", (req, res) => { - res.send("Hello, world!"); - }); - - // List endpoint - returns all available evaluators and their metadata - app.get( - "/list", - checkAuthorized, - asyncHandler(async (req, res) => { - const evalDefs: EvaluatorDefinitions = {}; - - for (const [name, evaluator] of Object.entries(allEvaluators)) { - let parameters: SerializedParametersContainer | undefined; - - if (evaluator.parameters) { - const resolvedParams = await Promise.resolve(evaluator.parameters); - parameters = serializeRemoteEvalParametersContainer(resolvedParams); - } - - evalDefs[name] = { - parameters, - scores: (evaluator.scores ?? []).map((score, idx) => ({ - name: scorerName(score, idx), - })), - classifiers: (evaluator.classifiers ?? []).map((classifier, idx) => ({ - name: classifierName(classifier, idx), - })), - }; - } - - res.json(evalDefs); - }), - ); - - app.post( - "/eval", - checkAuthorized, - asyncHandler(async (req, res) => { - const { - name, - parameters, - parent, - experiment_name, - project_id, - data, - scores, - stream, - } = evalBodySchema.parse(req.body); - - if (!req.ctx?.state) { - res - .status(500) - .json({ error: "Braintrust state not initialized in request" }); - return; - } - const state = req.ctx.state; - - const evaluator = allEvaluators[name]; - if (!evaluator) { - res.status(404).json({ error: `Evaluator '${name}' not found` }); - return; - } - - if (evaluator.parameters) { - try { - await validateParameters(parameters ?? {}, evaluator.parameters); - } catch (e) { - console.error("Error validating parameters", e); - if ( - e instanceof z.ZodError || - e instanceof ValidationError || - e instanceof Error - ) { - res.status(400).json({ - error: e.message, - }); - return; - } - throw e; - } - } - - const resolvedData = await getDataset(state, data); - const evalData = callEvaluatorData(resolvedData); - console.log("Starting eval", evaluator.evalName); - - // Set up SSE headers - if (stream) { - res.setHeader("Content-Type", "text/event-stream"); - res.setHeader("Cache-Control", "no-cache"); - res.setHeader("Connection", "keep-alive"); - } else { - res.setHeader("Content-Type", "application/json"); - } - - const task = async ( - input: unknown, - hooks: EvalHooks, - ) => { - const result = await evaluator.task(input, hooks); - - hooks.reportProgress({ - format: "code", - output_type: "completion", - event: "json_delta", - data: JSON.stringify(result), - }); - return result; - }; - - try { - const summary = await Eval( - "worker-thread", - { - ...evaluator, - data: evalData.data, - scores: (evaluator.scores ?? []).concat( - scores?.map((score) => - makeScorer( - state, - score.name, - score.function_id, - req.ctx?.projectId, - ), - ) ?? [], - ), - task, - state, - experimentName: experiment_name ?? undefined, - projectId: project_id ?? undefined, - }, - { - // Avoid printing the bar to the console. - progress: { - start: () => {}, - stop: () => { - console.log("Finished running experiment"); - }, - increment: () => {}, - }, - stream: (data: SSEProgressEventData) => { - if (stream) { - res.write( - serializeSSEEvent({ - event: "progress", - data: JSON.stringify(data), - }), - ); - } - }, - onStart: (metadata) => { - if (stream) { - res.write( - serializeSSEEvent({ - event: "start", - data: JSON.stringify(metadata), - }), - ); - } - }, - parent: parseParent(parent), - parameters: parameters ?? {}, - }, - ); - - if (stream) { - res.write( - serializeSSEEvent({ - event: "summary", - data: JSON.stringify(summary.summary), - }), - ); - res.write( - serializeSSEEvent({ - event: "done", - data: "", - }), - ); - } else { - res.json(summary.summary); - } - } catch (e) { - console.error("Error running eval", e); - if (stream) { - res.write( - serializeSSEEvent({ - event: "error", - data: JSON.stringify(e), - }), - ); - } else { - res.status(500).json({ error: e }); - } - } finally { - res.end(); - } - }), - ); - - app.use(errorHandler); - - // Start the server - app.listen(opts.port, opts.host, () => { - console.log(`Dev server running at http://${opts.host}:${opts.port}`); - }); -} -const asyncHandler = - (fn: (req: Request, res: Response, next: NextFunction) => Promise) => - (req: Request, res: Response, next: NextFunction) => { - Promise.resolve(fn(req, res, next)).catch(next); - }; - -async function getDataset( - state: BraintrustState, - data: RunEvalRequest["data"], -): Promise> { - if ("project_name" in data) { - return initDataset({ - state, - project: data.project_name, - dataset: data.dataset_name, - version: data.dataset_version ?? undefined, - environment: data.dataset_environment ?? undefined, - _internal_btql: data._internal_btql ?? undefined, - }); - } else if ("dataset_id" in data) { - const datasetInfo = await getDatasetById({ - state, - datasetId: data.dataset_id, - }); - return initDataset({ - state, - projectId: datasetInfo.projectId, - dataset: datasetInfo.dataset, - version: data.dataset_version ?? undefined, - environment: data.dataset_environment ?? undefined, - _internal_btql: data._internal_btql ?? undefined, - }); - } else if ("experiment_name" in data) { - return BaseExperiment({ name: data.experiment_name }); - } else { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return data.data as EvalCase[]; - } -} - -const datasetFetchSchema = z.object({ - project_id: z.string(), - name: z.string(), -}); -async function getDatasetById({ - state, - datasetId, -}: { - state: BraintrustState; - datasetId: string; -}): Promise<{ projectId: string; dataset: string }> { - const dataset = await state.appConn().post_json("api/dataset/get", { - id: datasetId, - }); - const parsed = z.array(datasetFetchSchema).parse(dataset); - if (parsed.length === 0) { - throw new Error(`Dataset '${datasetId}' not found`); - } - return { projectId: parsed[0].project_id, dataset: parsed[0].name }; -} - -function makeScorer( - state: BraintrustState, - name: string, - score: FunctionId, - projectId: string | undefined, -): EvalScorer { - const ret = async (input: EvalCase) => { - const request: InvokeFunctionRequest = { - ...score, - input, - parent: await getSpanParentObject().export(), - stream: false, - mode: "auto", - strict: true, - }; - const headers: Record = { - Accept: "application/json", - }; - if (projectId) { - headers["x-bt-project-id"] = projectId; - } - const result = await state.proxyConn().post(`function/invoke`, request, { - headers, - }); - const data = await result.json(); - // NOTE: Ideally we can parse this value with a zod schema. - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return data as OneOrMoreScores; - }; - - Object.defineProperties(ret, { - name: { value: `Remote eval scorer (${name})` }, - }); - - return ret; -} - -export const _exportsForTestingOnly = { - getDataset, -}; diff --git a/js/dev/stream.ts b/js/dev/stream.ts deleted file mode 100644 index 7bca3612b..000000000 --- a/js/dev/stream.ts +++ /dev/null @@ -1,14 +0,0 @@ -export interface RawSSEEvent { - id?: string; - event?: string; - data: string; -} - -export function serializeSSEEvent(event: RawSSEEvent): string { - return ( - Object.entries(event) - .filter(([_key, value]) => value !== undefined) - .map(([key, value]) => `${key}: ${value}`) - .join("\n") + "\n\n" - ); -} diff --git a/js/dev/types.ts b/js/dev/types.ts deleted file mode 100644 index da992efea..000000000 --- a/js/dev/types.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { - FunctionId as functionIdSchema, - InvokeParent as invokeParentSchema, - RunEval as runEvalSchema, - PromptData as promptDataSchema, -} from "../src/generated_types"; -import { z } from "zod/v3"; -import { EvaluatorDef } from "../src/framework"; -import { BaseMetadata } from "../src/logger"; - -export const evalBodySchema = z.object({ - name: z.string(), - parameters: z.record(z.string(), z.unknown()).nullish(), - data: runEvalSchema.shape.data, - scores: z - .array( - z.object({ - function_id: functionIdSchema, - name: z.string(), - }), - ) - .nullish(), - experiment_name: z.string().nullish(), - project_id: z.string().nullish(), - parent: invokeParentSchema.optional(), - stream: z.boolean().optional(), -}); - -export type EvaluatorManifest = Record< - string, - EvaluatorDef ->; - -export const staticParametersSchema = z.record( - z.string(), - z.union([ - z.object({ - type: z.literal("prompt"), - default: promptDataSchema.optional(), - description: z.string().optional(), - }), - z.object({ - type: z.literal("model"), - default: z.string().optional(), - description: z.string().optional(), - }), - z.object({ - type: z.literal("data"), - schema: z.record(z.unknown()), - default: z.unknown().optional(), - description: z.string().optional(), - }), - ]), -); - -export type StaticParametersSchema = z.infer; - -const evalParametersSerializedSchema = staticParametersSchema; -export type EvalParameterSerializedSchema = z.infer< - typeof evalParametersSerializedSchema ->; - -export const parametersSchema = z.object({ - type: z.literal("object"), - properties: z.record(z.string(), z.record(z.unknown())), - required: z.array(z.string()).optional(), - additionalProperties: z.boolean().optional(), -}); - -export type ParametersSchema = z.infer; - -const parametersSourceSchema = z.object({ - parametersId: z.string().optional(), - slug: z.string(), - name: z.string(), - projectId: z.string().optional(), - version: z.string().optional(), -}); - -export type ParametersSource = z.infer; - -const parametersContainerSchema = z.object({ - type: z.literal("braintrust.parameters"), - schema: parametersSchema, - source: parametersSourceSchema, -}); - -export type ParametersContainer = z.infer; - -const staticParametersContainerSchema = z.object({ - type: z.literal("braintrust.staticParameters"), - schema: staticParametersSchema, - source: z.null().nullish(), -}); - -export type StaticParametersContainer = z.infer< - typeof staticParametersContainerSchema ->; - -export const serializedParametersContainerSchema = z.union([ - parametersContainerSchema, - staticParametersContainerSchema, - // keeping this type here since old versions of the SDK will still pass the unwrapped schema and we need to handle this in the app - staticParametersSchema, -]); - -export type SerializedParametersContainer = z.infer< - typeof serializedParametersContainerSchema ->; - -export const evaluatorDefinitionSchema = z.object({ - parameters: serializedParametersContainerSchema.nullish(), - scores: z.array(z.object({ name: z.string() })).optional(), - classifiers: z.array(z.object({ name: z.string() })).optional(), -}); -export type EvaluatorDefinition = z.infer; - -export const evaluatorDefinitionsSchema = z.record( - z.string(), - evaluatorDefinitionSchema, -); - -export type EvaluatorDefinitions = z.infer; diff --git a/js/eslint.config.ts b/js/eslint.config.ts index 04e2b8f78..80829409c 100644 --- a/js/eslint.config.ts +++ b/js/eslint.config.ts @@ -12,21 +12,18 @@ const tsupConfig = Array.isArray(tsupConfigImport) ? tsupConfigImport : tsupConfigImport.default || []; -// Extract entry points from tsup config, excluding CLI -const entryFiles = tsupConfig - .flatMap((config) => { - const entries = Array.isArray(config.entry) - ? config.entry - : Object.values(config.entry); - return entries; - }) - .filter((entry) => !entry.includes("cli")); +// Extract entry points from tsup config +const entryFiles = tsupConfig.flatMap((config) => { + const entries = Array.isArray(config.entry) + ? config.entry + : Object.values(config.entry); + return entries; +}); export default [ { ignores: [ "dist/**", - "dev/dist/**", "util/dist/**", "node_modules/**", "vendor/**", @@ -164,38 +161,6 @@ export default [ "no-restricted-properties": "off", }, }, - { - files: ["src/**/*.ts", "src/**/*.tsx"], - ignores: [ - "src/cli/**", - "src/debug-logger.ts", - "src/framework.ts", - "src/framework2.ts", - "src/isomorph.ts", - "src/sandbox.ts", - "src/template/**", - "src/reporters/**", - "src/prompt-cache/**", - "src/eval-parameters.ts", - "src/wrappers/**", - "src/instrumentation/**", - "src/auto-instrumentations/**", - ], - rules: { - "no-restricted-imports": [ - "error", - { - patterns: [ - { - group: ["**/cli", "**/cli/**", "./cli", "./cli/**"], - message: - "Importing from 'cli' directory is not allowed. CLI code should not be imported by SDK code.", - }, - ], - }, - ], - }, - }, { files: ["src/**/*.ts", "src/**/*.tsx"], ignores: [...entryFiles, "**/*.test.ts", "**/*.test.tsx"], diff --git a/js/package.json b/js/package.json index 2d0ca836e..2f9375376 100644 --- a/js/package.json +++ b/js/package.json @@ -19,7 +19,6 @@ "./dist/index.d.mts": "./dist/browser.d.mts" }, "bin": { - "braintrust": "./dist/cli.js", "bt": "./bin/bt" }, "exports": { @@ -77,12 +76,6 @@ "module": "./dist/index.mjs", "require": "./dist/index.js" }, - "./dev": { - "types": "./dev/dist/index.d.ts", - "import": "./dev/dist/index.mjs", - "module": "./dev/dist/index.mjs", - "require": "./dev/dist/index.js" - }, "./util": { "types": "./util/dist/index.d.ts", "import": "./util/dist/index.mjs", @@ -133,7 +126,6 @@ }, "files": [ "dist/**/*", - "dev/dist/**/*", "util/dist/**/*", "bin/bt", "scripts/bt-helper.js", @@ -146,9 +138,9 @@ "build": "cross-env NODE_OPTIONS=\"--max-old-space-size=8192\" tsup", "check:typings": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json", "watch": "tsup --watch", - "clean": "rm -r dist/* && rm -r dev/dist/* && rm -r util/dist/*", + "clean": "rm -r dist/* && rm -r util/dist/*", "docs": "typedoc --options typedoc.json src/node/index.ts", - "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"", + "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"", "test:core": "pnpm prune && pnpm test", "test:checks": "pnpm run test:core && pnpm run test:vitest", "test:all": "pnpm run test:checks", @@ -158,35 +150,26 @@ "test:ai-sdk-v1": "vitest run src/wrappers/ai-sdk-v1.test.ts", "test:ai-sdk-v2": "vitest run src/wrappers/ai-sdk-v2.test.ts src/wrappers/ai-sdk-v1.test.ts", "test:ai-sdk-v3": "vitest run src/wrappers/ai-sdk-v3.test.ts", - "test:zod-v3": "vitest run src/zod/zod-v3-serialization.test.ts", - "test:zod-v4": "vitest run src/zod/zod-v4-serialization.test.ts", "test:mastra": "vitest run src/wrappers/mastra.test.ts", "test:vitest": "pnpm --filter @braintrust/vitest-wrapper-tests test", - "test:output": "tsx scripts/test-output.ts --with-comparison --with-metrics --with-progress", "bench": "tsx src/queue.bench.ts", "lint": "eslint .", "fix:lint": "eslint --fix .", "playground": "tsx playground.ts", "playground:auto": "mkdir -p .context && pnpm exec esbuild playground.ts --platform=node --format=esm --outfile=.context/playground.auto.mjs && node --import ./dist/auto-instrumentations/hook.mjs ./.context/playground.auto.mjs", - "playground:cli:push": "node dist/cli.js push playground.ts", - "playground:cli:eval": "node dist/cli.js eval playground.ts", "yalc:publish": "yalc publish" }, "author": "", "license": "MIT", + "engines": { + "node": "^20.12.0 || >=22.13.0" + }, "devDependencies": { "@anthropic-ai/sdk": "^0.60.0", - "@nodelib/fs.walk": "^1.2.8", - "@types/argparse": "^2.0.14", "@types/async": "^3.2.24", - "@types/cli-progress": "^3.11.5", - "@types/cors": "^2.8.17", "@types/esquery": "^1.5.4", - "@types/express": "^5.0.0", - "@types/http-errors": "^2.0.4", "@types/mustache": "^4.2.5", "@types/node": "^20.10.5", - "@types/pluralize": "^0.0.30", "@typescript-eslint/eslint-plugin": "^8.49.0", "@typescript-eslint/parser": "^8.49.0", "ai": "^6.0.0", @@ -213,31 +196,21 @@ "zod": "^3.25.34" }, "dependencies": { - "@next/env": "^14.2.3", "@vercel/functions": "^1.0.2", "acorn": "^8.16.0", "acorn-import-attributes": "^1.9.5", "ajv": "^8.20.0", - "argparse": "^2.0.1", "astring": "^1.9.0", "cjs-module-lexer": "^2.2.0", - "cli-progress": "^3.12.0", - "cli-table3": "^0.6.5", - "cors": "^2.8.5", "dotenv": "^16.4.5", "esbuild": "0.28.1", "esquery": "^1.7.0", "eventsource-parser": "^1.1.2", - "express": "^5.2.1", - "http-errors": "^2.0.0", "meriyah": "^6.1.4", - "minimatch": "^10.2.5", "module-details-from-path": "^1.0.4", "mustache": "^4.2.0", - "pluralize": "^8.0.0", "semifies": "^1.0.0", "source-map": "^0.7.4", - "termi-link": "^1.0.1", "unplugin": "^2.3.5", "uuid": "^11.1.1", "zod-to-json-schema": "^3.25.0" diff --git a/js/scripts/test-output.ts b/js/scripts/test-output.ts deleted file mode 100644 index 8a9ca0858..000000000 --- a/js/scripts/test-output.ts +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env tsx - -/** - * Test script to preview CLI output formatting without running actual experiments. - * - * Usage: - * pnpm tsx scripts/test-output.ts - * pnpm tsx scripts/test-output.ts --with-comparison - * pnpm tsx scripts/test-output.ts --with-metrics - * pnpm tsx scripts/test-output.ts --with-progress - */ - -import { formatExperimentSummary } from "../src/framework"; -import type { ExperimentSummary } from "../src/logger"; -import { BarProgressReporter } from "../src/cli/progress"; - -function createSampleSummary(options: { - withComparison?: boolean; - withMetrics?: boolean; -}): ExperimentSummary { - const summary: ExperimentSummary = { - projectName: "my-project", - experimentName: "test-experiment", - experimentId: "exp-123", - projectId: "proj-456", - experimentUrl: - "https://www.braintrust.dev/app/my-org/p/my-project/experiments/test-experiment", - projectUrl: "https://www.braintrust.dev/app/my-org/p/my-project", - scores: { - accuracy: { - name: "Accuracy", - score: 0.95, - diff: options.withComparison ? 0.02 : undefined, - improvements: options.withComparison ? 15 : 0, - regressions: options.withComparison ? 3 : 0, - }, - factuality: { - name: "Factuality", - score: 0.87, - diff: options.withComparison ? -0.01 : undefined, - improvements: options.withComparison ? 8 : 0, - regressions: options.withComparison ? 12 : 0, - }, - }, - }; - - if (options.withMetrics) { - summary.metrics = { - duration: { - name: "Duration", - metric: 1.23, - unit: "s", - diff: options.withComparison ? -0.15 : undefined, - improvements: options.withComparison ? 10 : 0, - regressions: options.withComparison ? 5 : 0, - }, - llm_duration: { - name: "LLM duration", - metric: 0.45, - unit: "s", - diff: options.withComparison ? -0.08 : undefined, - improvements: options.withComparison ? 12 : 0, - regressions: options.withComparison ? 3 : 0, - }, - prompt_tokens: { - name: "Prompt tokens", - metric: 4282, - unit: "", - diff: options.withComparison ? 0.035 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - completion_tokens: { - name: "Completion tokens", - metric: 310, - unit: "", - diff: options.withComparison ? -0.08 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - total_tokens: { - name: "Total tokens", - metric: 4592, - unit: "", - diff: options.withComparison ? 0.027 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - estimated_cost: { - name: "Estimated cost", - metric: 0.01, - unit: "$", - diff: options.withComparison ? 0.2 : undefined, - improvements: options.withComparison ? 0 : 0, - regressions: options.withComparison ? 0 : 0, - }, - }; - } - - if (options.withComparison) { - summary.comparisonExperimentName = "baseline-experiment"; - } - - return summary; -} - -async function simulateProgress(evaluatorName: string, total: number) { - const progressReporter = new BarProgressReporter(); - progressReporter.start(evaluatorName, total); - - // Simulate progress with random delays - for (let i = 0; i < total; i++) { - await new Promise((resolve) => - setTimeout(resolve, 50 + Math.random() * 100), - ); - progressReporter.increment(evaluatorName); - } - - progressReporter.stop(); -} - -async function main() { - const args = process.argv.slice(2); - const withComparison = args.includes("--with-comparison"); - const withMetrics = args.includes("--with-metrics"); - const withProgress = args.includes("--with-progress"); - - console.log("\n" + "=".repeat(60)); - console.log("Testing CLI Output Formatting"); - console.log("=".repeat(60) + "\n"); - - // Test 1: Basic summary - console.log("1. Basic Summary (scores only):"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 10); - } - - const basicSummary = createSampleSummary({}); - console.log(formatExperimentSummary(basicSummary)); - console.log("\n"); - - // Test 2: With comparison - if (withComparison) { - console.log("2. Summary with Comparison:"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 15); - } - - const comparisonSummary = createSampleSummary({ withComparison: true }); - console.log(formatExperimentSummary(comparisonSummary)); - console.log("\n"); - } - - // Test 3: With metrics - if (withMetrics) { - console.log("3. Summary with Metrics:"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 20); - } - - const metricsSummary = createSampleSummary({ withMetrics: true }); - console.log(formatExperimentSummary(metricsSummary)); - console.log("\n"); - } - - // Test 4: Full summary - console.log("4. Full Summary (scores + metrics + comparison):"); - console.log("-".repeat(60)); - - if (withProgress) { - await simulateProgress("test-experiment", 25); - } - - const fullSummary = createSampleSummary({ - withComparison: true, - withMetrics: true, - }); - console.log(formatExperimentSummary(fullSummary)); - console.log("\n"); - - console.log("=".repeat(60)); - console.log( - "Done! Use --with-comparison, --with-metrics, and --with-progress flags to see more variations.", - ); - console.log("=".repeat(60) + "\n"); -} - -main(); diff --git a/js/smoke/shared/src/suites/import-verification.ts b/js/smoke/shared/src/suites/import-verification.ts index 492ba2f0e..cf422006b 100644 --- a/js/smoke/shared/src/suites/import-verification.ts +++ b/js/smoke/shared/src/suites/import-verification.ts @@ -54,9 +54,6 @@ export interface BraintrustModule { Eval?: unknown; EvalResultWithSummary?: unknown; Reporter?: unknown; - runEvaluator?: unknown; - buildLocalSummary?: unknown; - reportFailures?: unknown; defaultErrorScoreHandler?: unknown; // Tracing (REQUIRED) @@ -221,27 +218,6 @@ export const testEvalExports = register("testEvalExports", async (module) => { assertDefined(module.Reporter, "Reporter must exist"); assertType(module.Reporter, "function", "Reporter must be a function"); - assertDefined(module.runEvaluator, "runEvaluator must exist"); - assertType( - module.runEvaluator, - "function", - "runEvaluator must be a function", - ); - - assertDefined(module.buildLocalSummary, "buildLocalSummary must exist"); - assertType( - module.buildLocalSummary, - "function", - "buildLocalSummary must be a function", - ); - - assertDefined(module.reportFailures, "reportFailures must exist"); - assertType( - module.reportFailures, - "function", - "reportFailures must be a function", - ); - assertDefined( module.defaultErrorScoreHandler, "defaultErrorScoreHandler must exist", @@ -252,7 +228,7 @@ export const testEvalExports = register("testEvalExports", async (module) => { "defaultErrorScoreHandler must be a function", ); - return "Eval exports verified (7 runtime exports)"; + return "Eval exports verified (4 runtime exports)"; }); export const testTracingExports = register( diff --git a/js/src/auto-instrumentations/configs/ai-sdk.ts b/js/src/auto-instrumentations/configs/ai-sdk.ts index cdbb0c1f3..d38234e5e 100644 --- a/js/src/auto-instrumentations/configs/ai-sdk.ts +++ b/js/src/auto-instrumentations/configs/ai-sdk.ts @@ -2,7 +2,7 @@ import type { InstrumentationConfig } from "../orchestrion-js"; import { aiSDKChannels, harnessAgentChannels, -} from "../../instrumentation/plugins/ai-sdk-channels"; +} from "../../instrumentation/providers/ai-sdk-channels"; /** * Instrumentation configurations for the Vercel AI SDK. @@ -222,7 +222,7 @@ export const aiSDKConfigs: InstrumentationConfig[] = [ }, // AI SDK v7 exposes its telemetry lifecycle through a dispatcher created for - // each operation. We patch that dispatcher in the plugin instead of rewriting + // each operation. We patch that dispatcher in the consumer instead of rewriting // the module to call registerTelemetry(). { channelName: aiSDKChannels.v7CreateTelemetryDispatcher.channelName, diff --git a/js/src/auto-instrumentations/configs/all.ts b/js/src/auto-instrumentations/configs/all.ts index 23d58969a..3b42cb11a 100644 --- a/js/src/auto-instrumentations/configs/all.ts +++ b/js/src/auto-instrumentations/configs/all.ts @@ -13,7 +13,6 @@ import { cloudflareAgentsConfigs } from "./cloudflare-agents"; import { cloudflareThinkConfigs } from "./cloudflare-think"; import { cohereConfigs } from "./cohere"; import { cursorSDKConfigs } from "./cursor-sdk"; -import { flueConfigs } from "./flue"; import { genkitConfigs } from "./genkit"; import { gitHubCopilotConfigs } from "./github-copilot"; import { googleADKConfigs } from "./google-adk"; @@ -115,10 +114,6 @@ const defaultInstrumentationConfigGroups: readonly InstrumentationConfigGroup[] integrations: ["strandsAgentSDK"], configs: strandsAgentSDKConfigs, }, - { - integrations: ["flue"], - configs: flueConfigs, - }, { integrations: ["voyageai"], configs: voyageAIConfigs, diff --git a/js/src/auto-instrumentations/configs/anthropic.ts b/js/src/auto-instrumentations/configs/anthropic.ts index 3acaa4d75..862e3528b 100644 --- a/js/src/auto-instrumentations/configs/anthropic.ts +++ b/js/src/auto-instrumentations/configs/anthropic.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { anthropicChannels } from "../../instrumentation/plugins/anthropic-channels"; +import { anthropicChannels } from "../../instrumentation/providers/anthropic-channels"; /** * Instrumentation configurations for the Anthropic SDK. diff --git a/js/src/auto-instrumentations/configs/bedrock-runtime.ts b/js/src/auto-instrumentations/configs/bedrock-runtime.ts index 29b4d0f0d..91dddcfae 100644 --- a/js/src/auto-instrumentations/configs/bedrock-runtime.ts +++ b/js/src/auto-instrumentations/configs/bedrock-runtime.ts @@ -2,7 +2,7 @@ import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; import { smithyClientChannels, smithyCoreChannels, -} from "../../instrumentation/plugins/bedrock-runtime-channels"; +} from "../../instrumentation/providers/bedrock-runtime-channels"; export const bedrockRuntimeConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/claude-agent-sdk.ts b/js/src/auto-instrumentations/configs/claude-agent-sdk.ts index 1cf2c6537..6b663df61 100644 --- a/js/src/auto-instrumentations/configs/claude-agent-sdk.ts +++ b/js/src/auto-instrumentations/configs/claude-agent-sdk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { claudeAgentSDKChannels } from "../../instrumentation/plugins/claude-agent-sdk-channels"; +import { claudeAgentSDKChannels } from "../../instrumentation/providers/claude-agent-sdk-channels"; /** * Instrumentation configuration for the Claude Agent SDK. diff --git a/js/src/auto-instrumentations/configs/cloudflare-agents.ts b/js/src/auto-instrumentations/configs/cloudflare-agents.ts index 0b655bb78..b18372cb1 100644 --- a/js/src/auto-instrumentations/configs/cloudflare-agents.ts +++ b/js/src/auto-instrumentations/configs/cloudflare-agents.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cloudflareAgentsChannels } from "../../instrumentation/plugins/cloudflare-agents-channels"; +import { cloudflareAgentsChannels } from "../../instrumentation/providers/cloudflare-agents-channels"; const cloudflareAgentsVersionRange = ">=0.17.0 <0.18.0"; diff --git a/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts b/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts index 02f081612..1a603bffc 100644 --- a/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts +++ b/js/src/auto-instrumentations/configs/cloudflare-ai-chat.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cloudflareAIChatChannels } from "../../instrumentation/plugins/cloudflare-ai-chat-channels"; +import { cloudflareAIChatChannels } from "../../instrumentation/providers/cloudflare-ai-chat-channels"; const cloudflareAIChatVersionRange = ">=0.9.0 <0.10.0"; diff --git a/js/src/auto-instrumentations/configs/cloudflare-think.ts b/js/src/auto-instrumentations/configs/cloudflare-think.ts index 61fb4e8ae..d0e270b0f 100644 --- a/js/src/auto-instrumentations/configs/cloudflare-think.ts +++ b/js/src/auto-instrumentations/configs/cloudflare-think.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cloudflareThinkChannels } from "../../instrumentation/plugins/cloudflare-think-channels"; +import { cloudflareThinkChannels } from "../../instrumentation/providers/cloudflare-think-channels"; const cloudflareThinkVersionRange = ">=0.13.0 <0.14.0"; diff --git a/js/src/auto-instrumentations/configs/cohere.ts b/js/src/auto-instrumentations/configs/cohere.ts index 106980884..1ad936ac7 100644 --- a/js/src/auto-instrumentations/configs/cohere.ts +++ b/js/src/auto-instrumentations/configs/cohere.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cohereChannels } from "../../instrumentation/plugins/cohere-channels"; +import { cohereChannels } from "../../instrumentation/providers/cohere-channels"; export const cohereConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/cursor-sdk.ts b/js/src/auto-instrumentations/configs/cursor-sdk.ts index ecb845b4b..0b0eddb32 100644 --- a/js/src/auto-instrumentations/configs/cursor-sdk.ts +++ b/js/src/auto-instrumentations/configs/cursor-sdk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { cursorSDKChannels } from "../../instrumentation/plugins/cursor-sdk-channels"; +import { cursorSDKChannels } from "../../instrumentation/providers/cursor-sdk-channels"; const cursorSDKVersionRange = ">=1.0.7 <2.0.0"; diff --git a/js/src/auto-instrumentations/configs/flue.test.ts b/js/src/auto-instrumentations/configs/flue.test.ts deleted file mode 100644 index 456b5110f..000000000 --- a/js/src/auto-instrumentations/configs/flue.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { create } from "@apm-js-collab/code-transformer"; -import { describe, expect, it } from "vitest"; -import { flueConfigs, flueVersionRange } from "./flue"; - -describe("flue auto-instrumentation configs", () => { - it("targets the stable Flue 0.8 context factory", () => { - const matcher = create(flueConfigs); - const flue08Transformer = matcher.getTransformer( - "@flue/runtime", - "0.8.0", - "dist/internal.mjs", - ); - const flue10BetaTransformer = matcher.getTransformer( - "@flue/runtime", - "1.0.0-beta.3", - "dist/internal.mjs", - ); - const flue10Transformer = matcher.getTransformer( - "@flue/runtime", - "1.0.0", - "dist/internal.mjs", - ); - - expect(flueVersionRange).toBe(">=0.8.0 <1.0.0"); - expect(flue08Transformer).toBeDefined(); - expect(flue10BetaTransformer).toBeUndefined(); - expect(flue10Transformer).toBeUndefined(); - const transformed = flue08Transformer!.transform( - ` -function createFlueContext(config) { - return { config }; -} -`, - "esm", - ).code; - - expect(transformed).toContain( - "orchestrion:@flue/runtime:createFlueContext", - ); - }); - - it("does not target Flue content-hashed workflow or tool chunks", () => { - const matcher = create(flueConfigs); - - expect( - matcher.getTransformer( - "@flue/runtime", - "0.8.0", - "dist/handle-agent-DcUclCE2.mjs", - ), - ).toBeUndefined(); - expect( - matcher.getTransformer( - "@flue/runtime", - "0.8.0", - "dist/sandbox-DNEJXjr_.mjs", - ), - ).toBeUndefined(); - expect( - matcher.getTransformer("@flue/runtime", "1.0.0-beta.3", "dist/index.mjs"), - ).toBeUndefined(); - }); -}); diff --git a/js/src/auto-instrumentations/configs/flue.ts b/js/src/auto-instrumentations/configs/flue.ts deleted file mode 100644 index a04c02bf2..000000000 --- a/js/src/auto-instrumentations/configs/flue.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { InstrumentationConfig } from "../orchestrion-js"; -import { flueChannels } from "../../instrumentation/plugins/flue-channels"; - -export const flueVersionRange = ">=0.8.0 <1.0.0"; - -export const flueConfigs: InstrumentationConfig[] = [ - { - channelName: flueChannels.createContext.channelName, - module: { - name: "@flue/runtime", - versionRange: flueVersionRange, - filePath: "dist/internal.mjs", - }, - functionQuery: { - functionName: "createFlueContext", - kind: "Sync", - }, - }, -]; diff --git a/js/src/auto-instrumentations/configs/genkit.ts b/js/src/auto-instrumentations/configs/genkit.ts index 349635b01..5422a08fe 100644 --- a/js/src/auto-instrumentations/configs/genkit.ts +++ b/js/src/auto-instrumentations/configs/genkit.ts @@ -2,7 +2,7 @@ import type { InstrumentationConfig } from "../orchestrion-js"; import { genkitChannels, genkitCoreChannels, -} from "../../instrumentation/plugins/genkit-channels"; +} from "../../instrumentation/providers/genkit-channels"; const genkitVersionRange = ">=1.0.0 <2.0.0"; diff --git a/js/src/auto-instrumentations/configs/github-copilot.ts b/js/src/auto-instrumentations/configs/github-copilot.ts index 901f9e682..aed9eb7ed 100644 --- a/js/src/auto-instrumentations/configs/github-copilot.ts +++ b/js/src/auto-instrumentations/configs/github-copilot.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { gitHubCopilotChannels } from "../../instrumentation/plugins/github-copilot-channels"; +import { gitHubCopilotChannels } from "../../instrumentation/providers/github-copilot-channels"; export const gitHubCopilotConfigs: InstrumentationConfig[] = [ // ESM: CopilotClient.createSession diff --git a/js/src/auto-instrumentations/configs/google-adk.ts b/js/src/auto-instrumentations/configs/google-adk.ts index aeb72c44b..8c07fdbd7 100644 --- a/js/src/auto-instrumentations/configs/google-adk.ts +++ b/js/src/auto-instrumentations/configs/google-adk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { googleADKChannels } from "../../instrumentation/plugins/google-adk-channels"; +import { googleADKChannels } from "../../instrumentation/providers/google-adk-channels"; const googleADKVersionRange = ">=0.1.0"; const googleADKBundledIndexV06VersionRange = ">=0.6.1 <0.7.0"; diff --git a/js/src/auto-instrumentations/configs/google-genai.ts b/js/src/auto-instrumentations/configs/google-genai.ts index 16edd1d5c..53fb24d43 100644 --- a/js/src/auto-instrumentations/configs/google-genai.ts +++ b/js/src/auto-instrumentations/configs/google-genai.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { googleGenAIChannels } from "../../instrumentation/plugins/google-genai-channels"; +import { googleGenAIChannels } from "../../instrumentation/providers/google-genai-channels"; /** * Instrumentation configurations for the Google GenAI SDK. diff --git a/js/src/auto-instrumentations/configs/groq.ts b/js/src/auto-instrumentations/configs/groq.ts index 864b2a166..5db99b980 100644 --- a/js/src/auto-instrumentations/configs/groq.ts +++ b/js/src/auto-instrumentations/configs/groq.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { groqChannels } from "../../instrumentation/plugins/groq-channels"; +import { groqChannels } from "../../instrumentation/providers/groq-channels"; export const groqConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/huggingface-transformers.ts b/js/src/auto-instrumentations/configs/huggingface-transformers.ts index dd951ac25..781aab990 100644 --- a/js/src/auto-instrumentations/configs/huggingface-transformers.ts +++ b/js/src/auto-instrumentations/configs/huggingface-transformers.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { huggingFaceTransformersChannels } from "../../instrumentation/plugins/huggingface-transformers-channels"; +import { huggingFaceTransformersChannels } from "../../instrumentation/providers/huggingface-transformers-channels"; const moduleFiles = [ { diff --git a/js/src/auto-instrumentations/configs/huggingface.ts b/js/src/auto-instrumentations/configs/huggingface.ts index f4c9bec06..53b4d011f 100644 --- a/js/src/auto-instrumentations/configs/huggingface.ts +++ b/js/src/auto-instrumentations/configs/huggingface.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { huggingFaceChannels } from "../../instrumentation/plugins/huggingface-channels"; +import { huggingFaceChannels } from "../../instrumentation/providers/huggingface-channels"; export const huggingFaceConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/langchain.ts b/js/src/auto-instrumentations/configs/langchain.ts index e8108ed02..c1263a285 100644 --- a/js/src/auto-instrumentations/configs/langchain.ts +++ b/js/src/auto-instrumentations/configs/langchain.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { langChainChannels } from "../../instrumentation/plugins/langchain-channels"; +import { langChainChannels } from "../../instrumentation/providers/langchain-channels"; const langChainCoreVersionRange = ">=0.3.42"; const langChainCallbackManagerFilePath = "dist/callbacks/manager.js"; diff --git a/js/src/auto-instrumentations/configs/langsmith.ts b/js/src/auto-instrumentations/configs/langsmith.ts index 5ecc54f87..0904bc64c 100644 --- a/js/src/auto-instrumentations/configs/langsmith.ts +++ b/js/src/auto-instrumentations/configs/langsmith.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { langSmithChannels } from "../../instrumentation/plugins/langsmith-channels"; +import { langSmithChannels } from "../../instrumentation/providers/langsmith-channels"; const versionRange = ">=0.3.30 <1.0.0"; diff --git a/js/src/auto-instrumentations/configs/mistral.ts b/js/src/auto-instrumentations/configs/mistral.ts index e61fc24be..dab2493c3 100644 --- a/js/src/auto-instrumentations/configs/mistral.ts +++ b/js/src/auto-instrumentations/configs/mistral.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { mistralChannels } from "../../instrumentation/plugins/mistral-channels"; +import { mistralChannels } from "../../instrumentation/providers/mistral-channels"; export const mistralConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/ollama.ts b/js/src/auto-instrumentations/configs/ollama.ts index 5ef28eb64..8403bb652 100644 --- a/js/src/auto-instrumentations/configs/ollama.ts +++ b/js/src/auto-instrumentations/configs/ollama.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { ollamaChannels } from "../../instrumentation/plugins/ollama-channels"; +import { ollamaChannels } from "../../instrumentation/providers/ollama-channels"; const methods = [ ["chat", ollamaChannels.chat.channelName], diff --git a/js/src/auto-instrumentations/configs/openai-agents.ts b/js/src/auto-instrumentations/configs/openai-agents.ts index 3647edbc5..8bf95e99f 100644 --- a/js/src/auto-instrumentations/configs/openai-agents.ts +++ b/js/src/auto-instrumentations/configs/openai-agents.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openAIAgentsCoreChannels } from "../../instrumentation/plugins/openai-agents-channels"; +import { openAIAgentsCoreChannels } from "../../instrumentation/providers/openai-agents-channels"; const lifecycleMethods = [ ["onTraceStart", openAIAgentsCoreChannels.onTraceStart.channelName], diff --git a/js/src/auto-instrumentations/configs/openai-codex.ts b/js/src/auto-instrumentations/configs/openai-codex.ts index 40ee60784..a725caeb9 100644 --- a/js/src/auto-instrumentations/configs/openai-codex.ts +++ b/js/src/auto-instrumentations/configs/openai-codex.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openAICodexChannels } from "../../instrumentation/plugins/openai-codex-channels"; +import { openAICodexChannels } from "../../instrumentation/providers/openai-codex-channels"; const openAICodexVersionRange = ">=0.128.0 <1.0.0"; diff --git a/js/src/auto-instrumentations/configs/openai.ts b/js/src/auto-instrumentations/configs/openai.ts index 28d85b7f6..411bd2654 100644 --- a/js/src/auto-instrumentations/configs/openai.ts +++ b/js/src/auto-instrumentations/configs/openai.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openAIChannels } from "../../instrumentation/plugins/openai-channels"; +import { openAIChannels } from "../../instrumentation/providers/openai-channels"; /** * Instrumentation configurations for the OpenAI SDK. diff --git a/js/src/auto-instrumentations/configs/openrouter-agent.ts b/js/src/auto-instrumentations/configs/openrouter-agent.ts index f9f6191e1..d228c1ed8 100644 --- a/js/src/auto-instrumentations/configs/openrouter-agent.ts +++ b/js/src/auto-instrumentations/configs/openrouter-agent.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openRouterAgentChannels } from "../../instrumentation/plugins/openrouter-agent-channels"; +import { openRouterAgentChannels } from "../../instrumentation/providers/openrouter-agent-channels"; export const openRouterAgentConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/openrouter.ts b/js/src/auto-instrumentations/configs/openrouter.ts index c7b2ce7e4..2607b3da4 100644 --- a/js/src/auto-instrumentations/configs/openrouter.ts +++ b/js/src/auto-instrumentations/configs/openrouter.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { openRouterChannels } from "../../instrumentation/plugins/openrouter-channels"; +import { openRouterChannels } from "../../instrumentation/providers/openrouter-channels"; export const openRouterConfigs: InstrumentationConfig[] = [ { diff --git a/js/src/auto-instrumentations/configs/pi-coding-agent.ts b/js/src/auto-instrumentations/configs/pi-coding-agent.ts index 0bf6e2a56..3401aec0d 100644 --- a/js/src/auto-instrumentations/configs/pi-coding-agent.ts +++ b/js/src/auto-instrumentations/configs/pi-coding-agent.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { piCodingAgentChannels } from "../../instrumentation/plugins/pi-coding-agent-channels"; +import { piCodingAgentChannels } from "../../instrumentation/providers/pi-coding-agent-channels"; const piCodingAgentVersionRange = ">=0.79.0 <0.82.0"; diff --git a/js/src/auto-instrumentations/configs/strands-agent-sdk.ts b/js/src/auto-instrumentations/configs/strands-agent-sdk.ts index 7a7f7a5b9..f02439a19 100644 --- a/js/src/auto-instrumentations/configs/strands-agent-sdk.ts +++ b/js/src/auto-instrumentations/configs/strands-agent-sdk.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "@apm-js-collab/code-transformer"; -import { strandsAgentSDKChannels } from "../../instrumentation/plugins/strands-agent-sdk-channels"; +import { strandsAgentSDKChannels } from "../../instrumentation/providers/strands-agent-sdk-channels"; const strandsAgentSDKVersionRange = ">=1.0.0 <2.0.0"; diff --git a/js/src/auto-instrumentations/configs/voyageai.ts b/js/src/auto-instrumentations/configs/voyageai.ts index f54edf84e..9fbe886be 100644 --- a/js/src/auto-instrumentations/configs/voyageai.ts +++ b/js/src/auto-instrumentations/configs/voyageai.ts @@ -1,5 +1,5 @@ import type { InstrumentationConfig } from "../orchestrion-js"; -import { voyageAIChannels } from "../../instrumentation/plugins/voyageai-channels"; +import { voyageAIChannels } from "../../instrumentation/providers/voyageai-channels"; const GENERATED_CLIENT_MODULES = [ "dist/cjs/Client.js", diff --git a/js/src/auto-instrumentations/loader/cjs-patch.ts b/js/src/auto-instrumentations/loader/cjs-patch.ts index df5c448ae..d94511563 100644 --- a/js/src/auto-instrumentations/loader/cjs-patch.ts +++ b/js/src/auto-instrumentations/loader/cjs-patch.ts @@ -50,7 +50,7 @@ export class ModulePatch { // Per-package source patches (see loader/special-case-patches.ts). // Anti-pattern intentionally isolated in its own module — do not - // expand inline here; new integrations belong in the standard plugin + // expand inline here; new integrations belong in the standard consumer // pipeline. const patched = applySpecialCasePatch({ packageName, diff --git a/js/src/auto-instrumentations/loader/special-case-patches.ts b/js/src/auto-instrumentations/loader/special-case-patches.ts index f42215592..bf94b5926 100644 --- a/js/src/auto-instrumentations/loader/special-case-patches.ts +++ b/js/src/auto-instrumentations/loader/special-case-patches.ts @@ -6,8 +6,8 @@ * Every entry in this file represents a target SDK that doesn't expose a * stable extension point we can hook through global instrumentation hooks + the * internal Orchestrion matcher. New integrations should - * **prefer the standard channel-handler / `BasePlugin` pattern** used by - * every other integration in `js/src/instrumentation/plugins/*-plugin.ts`. + * **prefer the standard channel-consumer pattern** used by the integrations in + * `js/src/instrumentation/providers/*-plugin.ts`. * Only add an entry here when the target SDK gives us no other option (e.g. * the SDK relies on side-effectful module initialization, exposes its API * exclusively as re-exports from content-hashed chunks, or otherwise can't diff --git a/js/src/cli/auto-instrumentation.test.ts b/js/src/cli/auto-instrumentation.test.ts deleted file mode 100644 index 0b9f41495..000000000 --- a/js/src/cli/auto-instrumentation.test.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import * as fs from "node:fs/promises"; -import * as os from "node:os"; -import * as path from "node:path"; -import { execFile } from "node:child_process"; -import { promisify } from "node:util"; -import { fileURLToPath } from "node:url"; -import { newGlobalTracingChannel } from "../global-instrumentation-hooks"; -import { initializeHandles } from "./index"; -import type { FileHandle } from "./types"; - -vi.mock("./functions/load-module", () => ({ - loadModule: () => ({ - evaluators: {}, - functions: [], - parameters: [], - prompts: [], - reporters: {}, - }), -})); - -const googleGenAIChannel = "orchestrion:@google/genai:models.generateContent"; -const anthropicChannel = "orchestrion:@anthropic-ai/sdk:messages.create"; -const execFileAsync = promisify(execFile); -const loadModulePath = fileURLToPath( - new URL("./functions/load-module.ts", import.meta.url), -); -const globalHooksPath = fileURLToPath( - new URL("../global-instrumentation-hooks.ts", import.meta.url), -); -const debugLoggerPath = fileURLToPath( - new URL("../debug-logger.ts", import.meta.url), -); - -describe("eval auto-instrumentation", () => { - let fixtureDir: string; - let handles: Record = {}; - - beforeEach(async () => { - fixtureDir = await fs.mkdtemp( - path.join(os.tmpdir(), "braintrust-eval-instrumentation-"), - ); - await writeFixturePackages(fixtureDir); - }); - - afterEach(async () => { - vi.unstubAllEnvs(); - await Promise.all(Object.values(handles).map((handle) => handle.destroy())); - await Promise.all( - [ - ...new Set( - Object.values(handles) - .map((handle) => handle.bundleFile) - .filter( - (bundleFile): bundleFile is string => bundleFile !== undefined, - ) - .map((bundleFile) => path.dirname(bundleFile)), - ), - ].map((directory) => fs.rm(directory, { recursive: true, force: true })), - ); - handles = {}; - await fs.rm(fixtureDir, { recursive: true, force: true }); - }); - - it.each([ - [ - "directly", - `export { Models } from "@google/genai";`, - (exports: Record) => { - const Models = exports.Models as new () => { - generateContentInternal(input: string): Promise<{ text: string }>; - }; - return new Models().generateContentInternal("payload"); - }, - ], - [ - "through a dependency", - `export { invoke } from "google-genai-client";`, - (exports: Record) => - (exports.invoke as (input: string) => Promise<{ text: string }>)( - "payload", - ), - ], - ])( - "instruments @google/genai when imported %s", - async (_label, source, invoke) => { - const output = await buildEval(fixtureDir, source); - - expect(output).toContain(googleGenAIChannel); - - const lifecycle: string[] = []; - const hook = newGlobalTracingChannel(googleGenAIChannel); - const handlers = { - asyncEnd: () => lifecycle.push("asyncEnd"), - asyncStart: () => lifecycle.push("asyncStart"), - end: () => lifecycle.push("end"), - start: () => lifecycle.push("start"), - }; - hook.subscribe(handlers); - - try { - const loadedModule = { exports: {} as Record }; - Function( - "module", - "exports", - output, - )(loadedModule, loadedModule.exports); - - await expect(invoke(loadedModule.exports)).resolves.toEqual({ - text: "payload", - }); - expect(lifecycle).toEqual(["start", "end", "asyncStart", "asyncEnd"]); - } finally { - hook.unsubscribe(handlers); - } - }, - ); - - it("respects BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { - vi.stubEnv("BRAINTRUST_DISABLE_INSTRUMENTATION", "google-genai"); - - const output = await buildEval( - fixtureDir, - `export { Models } from "@google/genai";`, - ); - - expect(output).not.toContain(googleGenAIChannel); - }); - - it("instruments the final uploaded bundle", async () => { - await buildEval(fixtureDir, `export { Models } from "@google/genai";`); - - await handles[path.join(fixtureDir, "instrumentation.eval.ts")].bundle(); - const output = await fs.readFile( - handles[path.join(fixtureDir, "instrumentation.eval.ts")].bundleFile!, - "utf8", - ); - - expect(output).toContain(googleGenAIChannel); - }); - - it.each([ - ["instruments", undefined, ["start", "end", "asyncStart", "asyncEnd"]], - ["respects opt-out for", "anthropic", []], - ])("%s external eval dependencies", async (_label, disabled, expected) => { - const runnerPath = path.join(fixtureDir, "external-eval-runner.cjs"); - const evalFile = path.join(fixtureDir, "external.eval.ts"); - await fs.writeFile(evalFile, ""); - await fs.writeFile( - runnerPath, - `require("tsx/cjs"); -require("node:module").register = () => {}; -const { loadModule } = require(${JSON.stringify(loadModulePath)}); -const { newGlobalTracingChannel } = require(${JSON.stringify(globalHooksPath)}); -const lifecycle = []; -const channel = newGlobalTracingChannel(${JSON.stringify(anthropicChannel)}); -const handlers = Object.fromEntries( - ["start", "end", "asyncStart", "asyncEnd"].map((name) => [name, () => lifecycle.push(name)]), -); -channel.subscribe(handlers); -loadModule({ - inFile: ${JSON.stringify(evalFile)}, - moduleText: 'const { Messages } = require("@anthropic-ai/sdk/resources/messages/messages.js"); globalThis.__externalEvalResult = new Messages().create("payload");', -}); -Promise.resolve(globalThis.__externalEvalResult).then((result) => { - channel.unsubscribe(handlers); - process.stdout.write(JSON.stringify({ lifecycle, result })); -});`, - ); - - const env = { ...process.env }; - if (disabled) { - env.BRAINTRUST_DISABLE_INSTRUMENTATION = disabled; - } else { - delete env.BRAINTRUST_DISABLE_INSTRUMENTATION; - } - const { stdout } = await execFileAsync(process.execPath, [runnerPath], { - env, - }); - - expect(JSON.parse(stdout)).toEqual({ - lifecycle: expected, - result: { text: "payload" }, - }); - }); - - it("reports unexpected runtime instrumentation failures", async () => { - const runnerPath = path.join( - fixtureDir, - "failed-instrumentation-runner.cjs", - ); - const evalFile = path.join(fixtureDir, "failed-instrumentation.eval.ts"); - await fs.writeFile(evalFile, ""); - await fs.writeFile( - runnerPath, - `require("tsx/cjs"); -require("node:module").register = () => { throw new Error("setup failed"); }; -require(${JSON.stringify(debugLoggerPath)}).setGlobalDebugLogLevel("warn"); -const { loadModule } = require(${JSON.stringify(loadModulePath)}); -loadModule({ inFile: ${JSON.stringify(evalFile)}, moduleText: "" }); -process.stdout.write("loaded");`, - ); - - const { stderr, stdout } = await execFileAsync( - process.execPath, - [runnerPath], - { - env: { ...process.env, BRAINTRUST_DEBUG_LOG_LEVEL: "warn" }, - }, - ); - - expect(stdout).toBe("loaded"); - expect(stderr).toContain( - "Failed to enable auto-instrumentation for external eval dependencies", - ); - expect(stderr).toContain("setup failed"); - }); - - async function buildEval(fixtureDir: string, source: string) { - const evalFile = path.join(fixtureDir, "instrumentation.eval.ts"); - await fs.writeFile(evalFile, source); - handles = await initializeHandles({ files: [evalFile], mode: "eval" }); - - const result = await handles[evalFile].rebuild(); - if (result.type !== "success") { - throw result.error; - } - - return result.result.outputFiles?.[0].text ?? ""; - } -}); - -async function writeFixturePackages(fixtureDir: string) { - const googlePackageDir = path.join(fixtureDir, "node_modules/@google/genai"); - const indirectPackageDir = path.join( - fixtureDir, - "node_modules/google-genai-client", - ); - const anthropicPackageDir = path.join( - fixtureDir, - "node_modules/@anthropic-ai/sdk", - ); - await fs.mkdir(path.join(googlePackageDir, "dist/node"), { - recursive: true, - }); - await fs.mkdir(indirectPackageDir, { recursive: true }); - await fs.mkdir(path.join(anthropicPackageDir, "resources/messages"), { - recursive: true, - }); - - await Promise.all([ - fs.writeFile( - path.join(googlePackageDir, "package.json"), - JSON.stringify({ - name: "@google/genai", - version: "1.50.0", - type: "module", - exports: "./dist/node/index.mjs", - }), - ), - fs.writeFile( - path.join(googlePackageDir, "dist/node/index.mjs"), - `export class Models { - async generateContentInternal(input) { - return { text: input }; - } - }`, - ), - fs.writeFile( - path.join(indirectPackageDir, "package.json"), - JSON.stringify({ - name: "google-genai-client", - version: "1.0.0", - type: "module", - exports: "./index.mjs", - }), - ), - fs.writeFile( - path.join(indirectPackageDir, "index.mjs"), - `import { Models } from "@google/genai"; - export function invoke(input) { - return new Models().generateContentInternal(input); - }`, - ), - fs.writeFile( - path.join(anthropicPackageDir, "package.json"), - JSON.stringify({ - name: "@anthropic-ai/sdk", - version: "0.60.0", - exports: { - "./resources/messages/messages.js": - "./resources/messages/messages.js", - }, - }), - ), - fs.writeFile( - path.join(anthropicPackageDir, "resources/messages/messages.js"), - `class Messages { - async create(input) { - return { text: input }; - } - } - module.exports = { Messages };`, - ), - ]); -} diff --git a/js/src/cli/debug-logging.test.ts b/js/src/cli/debug-logging.test.ts deleted file mode 100644 index 49e352679..000000000 --- a/js/src/cli/debug-logging.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; - -vi.mock("@next/env", () => ({ - loadEnvConfig: vi.fn(), -})); - -vi.mock("dotenv", () => ({ - config: vi.fn(() => ({})), -})); - -vi.mock("./index", () => ({ - handleBuildFailure: vi.fn(), - initializeHandles: vi.fn(), -})); - -vi.mock("../logger", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - login: vi.fn().mockResolvedValue(undefined), - }; -}); - -import { login } from "../logger"; -import { resetDebugLoggingArgsForTests } from "./util/debug-logging"; -import { loadCLIEnv } from "./util/bundle"; - -describe("CLI debug logging", () => { - beforeEach(() => { - vi.clearAllMocks(); - resetDebugLoggingArgsForTests(); - }); - - test("loadCLIEnv forwards debugLogLevel to login", async () => { - await loadCLIEnv({ - verbose: false, - api_key: "test-key", - org_name: "test-org", - app_url: "https://braintrust.dev", - debug_logging: "debug", - }); - - expect(login).toHaveBeenCalledWith({ - apiKey: "test-key", - orgName: "test-org", - appUrl: "https://braintrust.dev", - debugLogLevel: "debug", - }); - }); - - test("loadCLIEnv treats --verbose as a deprecated alias for --debug-logging debug", async () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - - await loadCLIEnv({ - verbose: true, - api_key: "test-key", - org_name: "test-org", - app_url: "https://braintrust.dev", - }); - - expect(login).toHaveBeenCalledWith({ - apiKey: "test-key", - orgName: "test-org", - appUrl: "https://braintrust.dev", - debugLogLevel: "debug", - }); - expect(warnSpy).toHaveBeenCalledWith( - "Warning: --verbose is deprecated and will be removed in a future version of braintrust. Use --debug-logging debug to see full stack traces and troubleshooting details.", - ); - }); -}); diff --git a/js/src/cli/functions/infer-source.ts b/js/src/cli/functions/infer-source.ts deleted file mode 100644 index 0d480c621..000000000 --- a/js/src/cli/functions/infer-source.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { SourceMapConsumer } from "source-map"; -import * as fs from "node:fs/promises"; -import { EvaluatorFile, warning } from "../../framework"; -import { loadModule } from "./load-module"; -import { type CodeBundleType as CodeBundle } from "../../generated_types"; -import path from "node:path"; -import type { Node } from "typescript"; - -interface SourceMapContext { - inFiles: Record; - outFileModule: EvaluatorFile; - outFileLines: string[]; - sourceMapDir: string; - sourceMap: SourceMapConsumer; -} - -export async function makeSourceMapContext({ - inFile, - outFile, - sourceMapFile, -}: { - inFile: string; - outFile: string; - sourceMapFile: string; -}): Promise { - const [inFileContents, outFileContents, sourceMap] = await Promise.all([ - fs.readFile(inFile, "utf8"), - fs.readFile(outFile, "utf8"), - (async () => { - const sourceMap = await fs.readFile(sourceMapFile, "utf8"); - const sourceMapJSON = JSON.parse(sourceMap); - return new SourceMapConsumer(sourceMapJSON); - })(), - ]); - return { - inFiles: { [inFile]: inFileContents.split("\n") }, - outFileModule: loadModule({ inFile, moduleText: outFileContents }), - outFileLines: outFileContents.split("\n"), - sourceMapDir: path.dirname(sourceMapFile), - sourceMap, - }; -} - -// eslint-disable-next-line @typescript-eslint/no-unsafe-function-type -function isNative(fn: Function): boolean { - return /\{\s*\[native code\]\s*\}/.test(Function.prototype.toString.call(fn)); -} - -function locationToString(location: CodeBundle["location"]): string { - if (location.type === "experiment") { - return `eval ${location.eval_name} -> ${location.position.type}`; - } else if (location.type === "function") { - return `task ${location.index}`; - } else if (location.type === "sandbox") { - return `sandbox eval ${location.eval_name}`; - } else { - throw new Error(`Unknown location type`); - } -} - -export async function findCodeDefinition({ - location, - ctx: { inFiles, outFileModule, outFileLines, sourceMapDir, sourceMap }, -}: { - location: CodeBundle["location"]; - ctx: SourceMapContext; -}): Promise { - // eslint-disable-next-line @typescript-eslint/no-unsafe-function-type - let fn: Function | undefined = undefined; - - if (location.type === "experiment" || location.type === "sandbox") { - const evaluator = outFileModule.evaluators[location.eval_name]?.evaluator; - if (!evaluator) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Warning: failed to find evaluator for ${location.eval_name}. Will not display preview.`, - ), - ); - return undefined; - } - - if (location.type === "sandbox") { - fn = evaluator.task; - } else { - fn = - location.position.type === "task" - ? evaluator.task - : location.position.type === "scorer" - ? (evaluator.scores ?? [])[location.position.index] - : (evaluator.classifiers ?? [])[location.position.index]; - } - } else if (location.type === "function") { - fn = outFileModule.functions[location.index].handler; - } else { - throw new Error(`Unknown location type`); - } - - if (!fn) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Warning: failed to find ${locationToString(location)}. Will not display preview.`, - ), - ); - return undefined; - } - - const sourceCode = fn.toString(); - if (isNative(fn)) { - return undefined; - } - let lineNumber = 0; - let columnNumber = -1; - for (const line of outFileLines) { - const sourceDefinition = line.indexOf(sourceCode); - if (sourceDefinition !== -1) { - columnNumber = sourceDefinition; - break; - } - lineNumber++; - } - - if (columnNumber === -1) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Warning: failed to find code definition for ${fn.name}. Will not display preview.`, - ), - ); - return undefined; - } - const originalPosition = sourceMap.originalPositionFor({ - line: lineNumber + 1, - column: columnNumber + 1, - }); - - if (originalPosition.source === null || originalPosition.line === null) { - return undefined; - } - - if (!inFiles[originalPosition.source]) { - const originalFile = path.join(sourceMapDir, originalPosition.source); - inFiles[originalPosition.source] = ( - await fs.readFile(originalFile, "utf-8") - ).split("\n"); - } - - const originalLines = inFiles[originalPosition.source]; - - // Parse the file with Typescript to find the function definition - const ts = await getTsModule(); - if (!ts) { - return undefined; - } - const sourceFile = ts.createSourceFile( - originalPosition.source, - originalLines.join("\n"), - ts.ScriptTarget.Latest, - true, - ); - let functionNode: Node | undefined = undefined; - const targetPosition = ts.getPositionOfLineAndCharacter( - sourceFile, - originalPosition.line - 1, - originalPosition.column || 0, - ); - - ts.forEachChild(sourceFile, function visit(node) { - if (node.pos <= targetPosition && targetPosition < node.end) { - if ( - ts.isFunctionDeclaration(node) || - ts.isFunctionExpression(node) || - ts.isArrowFunction(node) - ) { - functionNode = node; - } else { - ts.forEachChild(node, visit); - } - } - }); - - if (!functionNode) { - return undefined; - } - - const printer = ts.createPrinter(); - const functionDefinition = printer.printNode( - ts.EmitHint.Unspecified, - functionNode, - sourceFile, - ); - - return functionDefinition; -} - -let tsModule: typeof import("typescript") | undefined = undefined; -async function getTsModule() { - if (!tsModule) { - try { - tsModule = require("typescript"); - } catch { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - "Failed to load TypeScript module. Will not use TypeScript to derive preview.", - ), - ); - } - } - return tsModule; -} diff --git a/js/src/cli/functions/load-module.ts b/js/src/cli/functions/load-module.ts deleted file mode 100644 index 1fa7296b0..000000000 --- a/js/src/cli/functions/load-module.ts +++ /dev/null @@ -1,54 +0,0 @@ -import nodeModulesPaths from "../jest/nodeModulesPaths"; -import path, { dirname } from "node:path"; -import { _internalGetGlobalState } from "../../logger"; -import { EvaluatorFile } from "../../framework"; -import { applyAutoInstrumentation } from "../../node/apply-auto-instrumentation"; -import { debugLogger } from "../../debug-logger"; - -function evalWithModuleContext(inFile: string, evalFn: () => T): T { - const modulePaths = [...module.paths]; - try { - module.paths = nodeModulesPaths(path.dirname(inFile), {}); - return evalFn(); - } finally { - module.paths = modulePaths; - } -} - -export function loadModule({ - inFile, - moduleText, -}: { - inFile: string; - moduleText: string; -}): EvaluatorFile { - return evalWithModuleContext(inFile, () => { - globalThis._evals = { - functions: [], - prompts: [], - parameters: [], - evaluators: {}, - reporters: {}, - }; - globalThis._lazy_load = true; - const state = _internalGetGlobalState(); - (globalThis as any)[Symbol.for("braintrust-state")] = state; - const __filename = inFile; - const __dirname = dirname(__filename); - try { - applyAutoInstrumentation(); - } catch (error) { - debugLogger.warn( - "Failed to enable auto-instrumentation for external eval dependencies; bundled dependencies remain instrumented:", - error, - ); - } - new Function("require", "module", "__filename", "__dirname", moduleText)( - require, - module, - __filename, - __dirname, - ); - return { ...globalThis._evals }; - }); -} diff --git a/js/src/cli/functions/upload.test.ts b/js/src/cli/functions/upload.test.ts deleted file mode 100644 index f0ecde997..000000000 --- a/js/src/cli/functions/upload.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { describe, expect, test, vi } from "vitest"; -import { buildBundledFunctionEntry } from "./upload"; - -vi.mock("./infer-source", () => ({ - findCodeDefinition: vi.fn(), - makeSourceMapContext: vi.fn(), -})); - -describe("buildBundledFunctionEntry", () => { - test("preserves tags and existing function fields", async () => { - const spec = { - project_id: "proj-123", - name: "test-tool", - slug: "test-tool", - description: "Test tool", - location: { - type: "function" as const, - index: 0, - }, - function_type: "tool" as const, - origin: { - object_type: "experiment" as const, - object_id: "exp-123", - internal: false, - }, - function_schema: { - parameters: { type: "object" }, - returns: { type: "string" }, - }, - if_exists: "replace" as const, - tags: ["ci", "sdk"], - metadata: { owner: "sdk" }, - }; - - const entry = await buildBundledFunctionEntry({ - spec, - runtime_context: { runtime: "node", version: "22.0.0" }, - bundleId: "bundle-123", - sourceMapContext: undefined, - }); - - expect(entry).toMatchObject({ - project_id: "proj-123", - name: "test-tool", - slug: "test-tool", - description: "Test tool", - origin: spec.origin, - function_type: "tool", - function_schema: spec.function_schema, - if_exists: "replace", - tags: ["ci", "sdk"], - metadata: { owner: "sdk" }, - function_data: { - type: "code", - data: { - type: "bundle", - runtime_context: { runtime: "node", version: "22.0.0" }, - location: spec.location, - bundle_id: "bundle-123", - }, - }, - }); - expect(((entry.function_data as any).data as any).preview).toBeUndefined(); - }); - - test("does not invent tags when they are omitted", async () => { - const entry = await buildBundledFunctionEntry({ - spec: { - project_id: "proj-123", - name: "test-tool", - slug: "test-tool", - description: "Test tool", - location: { - type: "function" as const, - index: 0, - }, - function_type: "tool" as const, - }, - runtime_context: { runtime: "node", version: "22.0.0" }, - bundleId: "bundle-123", - sourceMapContext: undefined, - }); - - expect(entry.tags).toBeUndefined(); - }); - - test("preserves classifier experiment locations", async () => { - const entry = await buildBundledFunctionEntry({ - spec: { - project_id: "proj-123", - name: "test-classifier", - slug: "test-classifier", - description: "Test classifier", - location: { - type: "experiment" as const, - eval_name: "eval-1", - position: { - type: "classifier" as const, - index: 0, - }, - }, - function_type: "classifier" as const, - }, - runtime_context: { runtime: "node", version: "22.0.0" }, - bundleId: "bundle-123", - sourceMapContext: undefined, - }); - - expect(entry.function_type).toBe("classifier"); - expect(((entry.function_data as any).data as any).location).toEqual({ - type: "experiment", - eval_name: "eval-1", - position: { - type: "classifier", - index: 0, - }, - }); - }); -}); diff --git a/js/src/cli/functions/upload.ts b/js/src/cli/functions/upload.ts deleted file mode 100644 index da513f503..000000000 --- a/js/src/cli/functions/upload.ts +++ /dev/null @@ -1,516 +0,0 @@ -import { - type CodeBundleType as CodeBundle, - type FunctionType as FunctionObject, - type IfExistsType as IfExists, -} from "../../generated_types"; -import type { BuildSuccess, FileHandle } from "../types"; -import { classifierName, scorerName, warning } from "../../framework"; -import { - _internalGetGlobalState, - Experiment, - FailedHTTPResponse, -} from "../../logger"; -import * as esbuild from "esbuild"; -import fs from "node:fs"; -import path from "node:path"; -import { createGzip } from "node:zlib"; -import { addAzureBlobHeaders, isEmpty } from "../../util"; -import { z } from "zod/v3"; -import { capitalize } from "../../../util/index"; -import { findCodeDefinition, makeSourceMapContext } from "./infer-source"; -import { slugify } from "../../../util/string_util"; -import { zodToJsonSchema } from "../../zod/utils"; -import pluralize from "pluralize"; -import { - FunctionEvent, - ProjectNameIdMap, - serializeRemoteEvalParametersContainer, -} from "../../framework2"; - -interface BundledFunctionSpec { - project_id: string; - name: string; - slug: string; - description: string; - location: CodeBundle["location"]; - function_type: FunctionObject["function_type"]; - origin?: FunctionObject["origin"]; - function_schema?: FunctionObject["function_schema"]; - if_exists?: IfExists; - tags?: string[]; - metadata?: Record; -} - -type BundledFunctionEntry = FunctionEvent & { - origin?: FunctionObject["origin"]; - function_schema?: FunctionObject["function_schema"]; -}; - -const SANDBOX_GROUP_NAME_METADATA_KEY = "_bt_sandbox_group_name"; - -const pathInfoSchema = z - .strictObject({ - url: z.string(), - bundleId: z.string(), - }) - .strip(); - -export async function uploadHandleBundles({ - buildResults, - evalToExperiment, - bundlePromises, - handles, - setCurrent, - showDetailedErrors, - defaultIfExists, -}: { - buildResults: BuildSuccess[]; - evalToExperiment?: Record>; - bundlePromises: { - [k: string]: Promise>; - }; - handles: Record; - showDetailedErrors: boolean; - setCurrent: boolean; - defaultIfExists: IfExists; -}) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Processing ${buildResults.length} ${pluralize("file", buildResults.length)}...`, - ); - - const projectNameToId = new ProjectNameIdMap(); - - const uploadPromises = buildResults.map(async (result) => { - if (result.type !== "success") { - return; - } - const sourceFile = result.sourceFile; - - const bundleSpecs: BundledFunctionSpec[] = []; - const prompts: FunctionEvent[] = []; - - if (setCurrent) { - for (let i = 0; i < result.evaluator.functions.length; i++) { - const fn = result.evaluator.functions[i]; - const project_id = await projectNameToId.resolve(fn.project); - - bundleSpecs.push({ - project_id: project_id, - name: fn.name, - slug: fn.slug, - description: fn.description ?? "", - function_type: fn.type, - location: { - type: "function", - index: i, - }, - function_schema: - fn.parameters || fn.returns - ? { - parameters: fn.parameters - ? zodToJsonSchema(fn.parameters) - : undefined, - returns: fn.returns ? zodToJsonSchema(fn.returns) : undefined, - } - : undefined, - if_exists: fn.ifExists, - tags: fn.tags, - metadata: fn.metadata, - }); - } - - for (const prompt of result.evaluator.prompts) { - prompts.push(await prompt.toFunctionDefinition(projectNameToId)); - } - - if (result.evaluator.parameters != null) { - for (const param of result.evaluator.parameters) { - prompts.push(await param.toFunctionDefinition(projectNameToId)); - } - } - } - - for (const evaluator of Object.values(result.evaluator.evaluators)) { - const experiment = - evalToExperiment?.[sourceFile]?.[evaluator.evaluator.evalName]; - - const baseInfo = { - project_id: experiment - ? (await experiment.project).id - : await projectNameToId.getId(evaluator.evaluator.projectName), - }; - - const namePrefix = setCurrent - ? evaluator.evaluator.experimentName - ? `${evaluator.evaluator.experimentName}` - : evaluator.evaluator.evalName - : experiment - ? `${await experiment.name}` - : evaluator.evaluator.evalName; - - const experimentId = experiment ? await experiment.id : undefined; - const origin: FunctionObject["origin"] = experimentId - ? { - object_type: "experiment", - object_id: experimentId, - internal: !setCurrent, - } - : undefined; - - const fileSpecs: BundledFunctionSpec[] = [ - { - ...baseInfo, - // There is a very small chance that someone names a function with the same convention, but - // let's assume it's low enough that it doesn't matter. - ...formatNameAndSlug(["eval", namePrefix, "task"]), - description: `Task for eval ${namePrefix}`, - location: { - type: "experiment", - eval_name: evaluator.evaluator.evalName, - position: { type: "task" }, - }, - function_type: "task", - origin, - }, - ...(evaluator.evaluator.scores ?? []).map( - (score, i): BundledFunctionSpec => { - const name = scorerName(score, i); - return { - ...baseInfo, - // There is a very small chance that someone names a function with the same convention, but - // let's assume it's low enough that it doesn't matter. - ...formatNameAndSlug(["eval", namePrefix, "scorer", name]), - description: `Score ${name} for eval ${namePrefix}`, - location: { - type: "experiment", - eval_name: evaluator.evaluator.evalName, - position: { type: "scorer", index: i }, - }, - function_type: "scorer", - origin, - }; - }, - ), - ...(evaluator.evaluator.classifiers ?? []).map( - (classifier, i): BundledFunctionSpec => { - const name = classifierName(classifier, i); - return { - ...baseInfo, - ...formatNameAndSlug(["eval", namePrefix, "classifier", name]), - description: `Classifier ${name} for eval ${namePrefix}`, - location: { - type: "experiment", - eval_name: evaluator.evaluator.evalName, - position: { type: "classifier", index: i }, - }, - function_type: "classifier", - origin, - }; - }, - ), - ]; - - bundleSpecs.push(...fileSpecs); - - if (setCurrent) { - const sourceStem = path - .basename(sourceFile, path.extname(sourceFile)) - .replace(/\.eval$/, ""); - const evalName = evaluator.evaluator.evalName; - const sandboxGroupName = sourceStem; - - const resolvedParameters = evaluator.evaluator.parameters - ? await Promise.resolve(evaluator.evaluator.parameters) - : undefined; - - const evaluatorDefinition = { - ...(resolvedParameters - ? { - parameters: - serializeRemoteEvalParametersContainer(resolvedParameters), - } - : {}), - scores: (evaluator.evaluator.scores ?? []).map((score, i) => ({ - name: scorerName(score, i), - })), - classifiers: (evaluator.evaluator.classifiers ?? []).map( - (classifier, i) => ({ - name: classifierName(classifier, i), - }), - ), - }; - - bundleSpecs.push({ - ...baseInfo, - name: `Eval ${evalName} sandbox`, - slug: slugify(`${sourceStem}-${evalName}-sandbox`), - description: `Sandbox eval ${evalName}`, - location: { - type: "sandbox", - sandbox_spec: { - provider: "lambda", - }, - entrypoints: [sourceFile], - eval_name: evalName, - evaluator_definition: evaluatorDefinition, - }, - function_type: "sandbox", - metadata: { - [SANDBOX_GROUP_NAME_METADATA_KEY]: sandboxGroupName, - }, - origin, - }); - } - } - - const slugs: Set = new Set(); - for (const spec of bundleSpecs) { - if (slugs.has(spec.slug)) { - throw new Error(`Duplicate slug: ${spec.slug}`); - } - slugs.add(spec.slug); - } - for (const prompt of prompts) { - if (slugs.has(prompt.slug)) { - throw new Error(`Duplicate slug: ${prompt.slug}`); - } - slugs.add(prompt.slug); - } - - return await uploadBundles({ - sourceFile, - prompts, - bundleSpecs, - bundlePromises, - handles, - defaultIfExists, - showDetailedErrors, - }); - }); - - const uploadResults = await Promise.all(uploadPromises); - const numUploaded = uploadResults.length; - const numFailed = uploadResults.filter((result) => !result).length; - - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `${numUploaded} ${pluralize("file", numUploaded)} uploaded ${ - numFailed > 0 - ? `with ${numFailed} error${numFailed > 1 ? "s" : ""}` - : "successfully" - }.`, - ); - - return { - numTotal: buildResults.length, - numUploaded, - numFailed, - }; -} - -async function uploadBundles({ - sourceFile, - prompts, - bundleSpecs, - bundlePromises, - handles, - defaultIfExists, - showDetailedErrors, -}: { - sourceFile: string; - prompts: FunctionEvent[]; - bundleSpecs: BundledFunctionSpec[]; - bundlePromises: { - [k: string]: Promise>; - }; - handles: Record; - defaultIfExists: IfExists; - showDetailedErrors: boolean; -}): Promise { - const orgId = _internalGetGlobalState().orgId; - if (!orgId) { - throw new Error("No organization ID found"); - } - - const loggerConn = _internalGetGlobalState().apiConn(); - const runtime_context = { - runtime: "node", - version: process.version.slice(1), - } as const; - - const bundle = await bundlePromises[sourceFile]; - const bundleFileName = handles[sourceFile].bundleFile; - if (!bundle || !bundleFileName) { - return false; - } - - const sourceMapContextPromise = makeSourceMapContext({ - inFile: sourceFile, - outFile: bundleFileName, - sourceMapFile: bundleFileName + ".map", - }); - - let pathInfo: z.infer | undefined = undefined; - if (bundleSpecs.length > 0) { - try { - pathInfo = pathInfoSchema.parse( - await loggerConn.post_json("function/code", { - org_id: orgId, - runtime_context, - }), - ); - } catch (e) { - if (showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(e); - } - const msg = - e instanceof FailedHTTPResponse - ? `Unable to upload your code. ${e.status} (${e.text}): ${e.data}` - : `Unable to upload your code. You most likely need to update the API: ${e}`; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(warning(msg)); - return false; - } - } - - // Upload bundleFile to pathInfo.url - if (isEmpty(bundleFileName)) { - throw new Error("No bundle file found"); - } - const bundleFile = path.resolve(bundleFileName); - const uploadPromise = (async (): Promise => { - if (!pathInfo) { - return true; - } - const bundleStream = fs.createReadStream(bundleFile).pipe(createGzip()); - const bundleData = await new Promise((resolve, reject) => { - const chunks: Buffer[] = []; - bundleStream.on("data", (chunk) => { - chunks.push(chunk); - }); - bundleStream.on("end", () => { - resolve(Buffer.concat(chunks)); - }); - bundleStream.on("error", reject); - }); - - const headers = { - "Content-Encoding": "gzip", - }; - addAzureBlobHeaders(headers, pathInfo.url); - - const resp = await fetch(pathInfo.url, { - method: "PUT", - body: bundleData, - headers, - }); - if (!resp.ok) { - throw new Error( - `Failed to upload bundle: ${resp.status} ${await resp.text()}`, - ); - } - return true; - })(); - - const sourceMapContext = await sourceMapContextPromise; - - // Insert the spec as prompt data - const functionEntries: FunctionEvent[] = [ - ...prompts, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - ...((await Promise.all( - bundleSpecs.map((spec) => - buildBundledFunctionEntry({ - spec, - runtime_context, - bundleId: pathInfo!.bundleId, - sourceMapContext, - }), - ), - )) as FunctionEvent[]), - ].map((fn) => ({ - ...fn, - if_exists: fn.if_exists ?? defaultIfExists, - })); - - const logPromise = (async (): Promise => { - try { - await _internalGetGlobalState().apiConn().post_json("insert-functions", { - functions: functionEntries, - }); - } catch (e) { - if (showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(e); - } - const msg = - e instanceof FailedHTTPResponse - ? `Failed to save function definitions for '${sourceFile}'. ${e.status} (${e.text}): ${e.data}` - : `Failed to save function definitions for '${sourceFile}'. You most likely need to update the API: ${e}`; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(warning(msg)); - return false; - } - return true; - })(); - - const [uploadSuccess, logSuccess] = await Promise.all([ - uploadPromise, - logPromise, - ]); - - return uploadSuccess && logSuccess; -} - -function formatNameAndSlug(pieces: string[]) { - const nonEmptyPieces = pieces.filter((piece) => piece.trim() !== ""); - return { - name: capitalize(nonEmptyPieces.join(" ")), - slug: slugify(nonEmptyPieces.join("-")), - }; -} - -export async function buildBundledFunctionEntry({ - spec, - runtime_context, - bundleId, - sourceMapContext, -}: { - spec: BundledFunctionSpec; - runtime_context: { - runtime: "node"; - version: string; - }; - bundleId: string; - sourceMapContext?: Awaited>; -}): Promise { - return { - project_id: spec.project_id, - name: spec.name, - slug: spec.slug, - description: spec.description, - function_data: { - type: "code", - data: { - type: "bundle", - runtime_context, - location: spec.location, - bundle_id: bundleId, - preview: sourceMapContext - ? await findCodeDefinition({ - location: spec.location, - ctx: sourceMapContext, - }) - : undefined, - }, - }, - origin: spec.origin, - function_type: spec.function_type ?? undefined, - function_schema: spec.function_schema, - if_exists: spec.if_exists, - tags: spec.tags, - metadata: spec.metadata, - }; -} diff --git a/js/src/cli/index.ts b/js/src/cli/index.ts deleted file mode 100755 index eb5004555..000000000 --- a/js/src/cli/index.ts +++ /dev/null @@ -1,1194 +0,0 @@ -#!/usr/bin/env node - -import * as esbuild from "esbuild"; -import * as dotenv from "dotenv"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import util, { styleText } from "node:util"; -import * as fsWalk from "@nodelib/fs.walk"; -import { minimatch } from "minimatch"; -import { ArgumentParser } from "argparse"; -import { v4 as uuidv4 } from "uuid"; -import pluralize from "pluralize"; -import { - login, - init as _initExperiment, - Experiment, - BaseMetadata, - Dataset, - type ParametersRef, - RemoteEvalParameters, -} from "../logger"; -import type { ProgressReporter } from "../reporters/types"; -import { - BarProgressReporter, - SimpleProgressReporter, -} from "./reporters/progress"; -import { terminalLink } from "termi-link"; - -// Re-use the module resolution logic from Jest -import type { ReporterDef } from "../reporters/types"; -import { - EvalData, - EvaluatorDef, - EvaluatorFile, - Filter, - callEvaluatorData, - error, - logError, - parseFilters, - runEvaluator, -} from "../framework"; -import { fancyReporter, warning } from "./reporters/eval"; -import { configureNode } from "../node/config"; -import { isEmpty } from "../util"; -import { loadEnvConfig } from "@next/env"; -import type { - BuildSuccess, - BuildFailure, - BtBuildResult, - FileHandle, - EvaluatorState, -} from "./types"; -import { uploadHandleBundles } from "./functions/upload"; -import { loadModule } from "./functions/load-module"; -import { bundleCommand } from "./util/bundle"; -import { RunArgs } from "./util/types"; -import { - normalizeDebugLoggingArgs, - shouldShowDetailedErrors, -} from "./util/debug-logging"; -import { pullCommand } from "./util/pull"; -import { runDevServer } from "../../dev/server"; -import { braintrustEsbuildPlugin } from "../auto-instrumentations/bundler/esbuild"; - -// This requires require -// https://stackoverflow.com/questions/50822310/how-to-import-package-json-in-typescript -const { version } = require("../../package.json"); - -// TODO: This could be loaded from configuration -const INCLUDE_EVAL = [ - "**/*.eval.ts", - "**/*.eval.tsx", - "**/*.eval.js", - "**/*.eval.jsx", -]; -const INCLUDE_BUNDLE = ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx"]; -const EXCLUDE = ["**/node_modules/**", "**/dist/**", "**/build/**"]; -const OUT_EXT = "js"; - -configureNode(); - -function evaluateBuildResults( - inFile: string, - buildResult: esbuild.BuildResult, -): EvaluatorFile | null { - if (!buildResult.outputFiles) { - return null; - } - const moduleText = buildResult.outputFiles[0].text; - return loadModule({ inFile, moduleText }); -} - -async function initExperiment( - evaluator: EvaluatorDef, - evaluatorData: { - data: EvalData; - baseExperiment: string | undefined; - }, -) { - const { data, baseExperiment: defaultBaseExperiment } = evaluatorData; - const parameters = await getExperimentParametersRef(evaluator.parameters); - // NOTE: This code is duplicated with initExperiment in js/src/framework.ts. - // Make sure to update that if you change this. - const logger = _initExperiment({ - state: evaluator.state, - ...(evaluator.projectId - ? { projectId: evaluator.projectId } - : { project: evaluator.projectName }), - experiment: evaluator.experimentName, - description: evaluator.description, - metadata: evaluator.metadata, - tags: evaluator.tags, - isPublic: evaluator.isPublic, - update: evaluator.update, - baseExperiment: evaluator.baseExperimentName ?? defaultBaseExperiment, - baseExperimentId: evaluator.baseExperimentId, - gitMetadataSettings: evaluator.gitMetadataSettings, - repoInfo: evaluator.repoInfo, - dataset: Dataset.isDataset(data) ? data : undefined, - parameters, - setCurrent: false, - }); - const info = await logger.summarize({ summarizeScores: false }); - const linkText = info.experimentUrl - ? terminalLink(info.experimentUrl, info.experimentUrl, { - fallback: (_text: string, url: string) => url, - }) - : "locally"; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - styleText("cyan", "▶") + - ` Experiment ${styleText("bold", info.experimentName)} is running at ${linkText}`, - ); - return logger; -} - -async function getExperimentParametersRef( - parameters: - | Record - | RemoteEvalParameters - | Promise> - | undefined, -): Promise { - if (!parameters) { - return undefined; - } - - const resolvedParameters = - parameters instanceof Promise ? await parameters : parameters; - - if (!RemoteEvalParameters.isParameters(resolvedParameters)) { - return undefined; - } - - if (resolvedParameters.id === undefined) { - return undefined; - } - - return { - id: resolvedParameters.id, - version: resolvedParameters.version, - }; -} - -function resolveReporter( - reporter: string | ReporterDef | undefined, - reporters: Record>, -) { - if (typeof reporter === "string") { - if (!reporters[reporter]) { - throw new Error(`Reporter ${reporter} not found`); - } - return reporters[reporter]; - } else if (!isEmpty(reporter)) { - return reporter; - } else if (Object.keys(reporters).length === 0) { - return fancyReporter; - } else if (Object.keys(reporters).length === 1) { - return reporters[Object.keys(reporters)[0]]; - } else { - const reporterNames = Object.keys(reporters).join(", "); - throw new Error( - `Multiple reporters found (${reporterNames}). Please specify a reporter explicitly.`, - ); - } -} - -type AllReports = Record< - string, - { - reporter: ReporterDef; - results: (unknown | Promise)[]; - } ->; - -function addReport( - evalReports: AllReports, - reporter: ReporterDef, - report: unknown, -) { - if (!evalReports[reporter.name]) { - evalReports[reporter.name] = { - reporter, - results: [], - }; - } - evalReports[reporter.name].results.push(report); -} - -function buildWatchPluginForEvaluator( - inFile: string, - opts: EvaluatorOpts, -): esbuild.Plugin { - const evaluators: EvaluatorState = { - evaluators: [], - reporters: {}, - }; - const plugin = { - name: "run-evalutator-on-end", - setup(build: esbuild.PluginBuild) { - build.onEnd(async (result) => { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Done building ${inFile}`); - - if (!result.outputFiles) { - if (opts.showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(`Failed to compile ${inFile}`); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(result.errors); - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(`Failed to compile ${inFile}: ${result.errors}`); - } - return; - } - - const evalResult = evaluateBuildResults(inFile, result); - if (!evalResult) { - return; - } - - evaluators.evaluators = evaluators.evaluators.filter( - (e) => e.sourceFile !== inFile, - ); - - // Update the evaluators and reporters - for (const evaluator of Object.values(evalResult.evaluators)) { - evaluators.evaluators.push({ - sourceFile: inFile, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - evaluator: evaluator.evaluator as EvaluatorDef< - unknown, - unknown, - unknown, - BaseMetadata - >, - reporter: evaluator.reporter, - }); - } - for (const [reporterName, reporter] of Object.entries( - evalResult.reporters, - )) { - evaluators.reporters[reporterName] = reporter; - } - - const evalReports: Record< - string, - { - reporter: ReporterDef; - results: unknown[]; - } - > = {}; - for (const evaluatorDef of Object.values(evalResult.evaluators)) { - const { evaluator, reporter } = evaluatorDef; - const evalData = callEvaluatorData(evaluator.data); - const logger = opts.noSendLogs - ? null - : await initExperiment(evaluator, evalData); - const evaluatorResult = await runEvaluator( - logger, - { - ...evaluator, - data: evalData.data, - }, - opts.progressReporter, - opts.filters, - undefined, - undefined, - ); - const resolvedReporter = resolveReporter( - reporter, - evaluators.reporters, // Let these accumulate across all files. - ); - - const report = resolvedReporter.reportEval( - evaluator, - evaluatorResult, - { - verbose: opts.showDetailedErrors, - jsonl: opts.jsonl, - }, - ); - - addReport(evalReports, resolvedReporter, report); - } - - for (const [reporterName, { reporter, results }] of Object.entries( - evalReports, - )) { - const success = await reporter.reportRun(await Promise.all(results)); - if (!success) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Reporter ${reporterName} failed.`)); - } - } - }); - }, - }; - - return plugin; -} - -async function initFile({ - inFile, - outFile, - bundleFile, - tsconfig, - plugins, - externalPackages, -}: { - inFile: string; - outFile: string; - bundleFile: string; - tsconfig?: string; - plugins?: PluginMaker[]; - externalPackages?: string[]; -}): Promise { - const buildOptions = buildOpts({ - fileName: inFile, - outFile, - tsconfig, - plugins, - externalPackages, - }); - const ctx = await esbuild.context(buildOptions); - - return { - inFile, - outFile, - bundleFile, - rebuild: async () => { - try { - const result = await ctx.rebuild(); - if (!result.outputFiles) { - return { - type: "failure", - error: new Error("No output file generated"), - sourceFile: inFile, - }; - } - const evaluator = evaluateBuildResults(inFile, result) || { - functions: [], - prompts: [], - parameters: [], - evaluators: {}, - reporters: {}, - }; - return { type: "success", result, evaluator, sourceFile: inFile }; - } catch (e) { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - return { type: "failure", error: e as Error, sourceFile: inFile }; - } - }, - bundle: async () => { - const buildOptions: esbuild.BuildOptions = { - ...buildOpts({ - fileName: inFile, - outFile: bundleFile, - tsconfig, - plugins: [], - externalPackages, - }), - external: ["fsevents", "chokidar"], - write: true, - minify: true, - sourcemap: true, - }; - return await esbuild.build(buildOptions); - }, - watch: () => { - ctx.watch(); - }, - destroy: async () => { - await ctx.dispose(); - }, - }; -} - -interface EvaluatorOpts { - showDetailedErrors: boolean; - apiKey?: string; - orgName?: string; - appUrl?: string; - noSendLogs: boolean; - bundle: boolean; - setCurrent: boolean; - terminateOnFailure: boolean; - watch: boolean; - list: boolean; - jsonl: boolean; - filters: Filter[]; - progressReporter: ProgressReporter; -} - -export function handleBuildFailure({ - result, - terminateOnFailure, - showDetailedErrors, -}: { - result: BuildFailure; - terminateOnFailure: boolean; - showDetailedErrors: boolean; -}) { - if (terminateOnFailure) { - throw result.error; - } else if (showDetailedErrors) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(`Failed to compile ${result.sourceFile}`); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(result.error); - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - `Failed to compile ${result.sourceFile}: ${result.error.message}`, - ); - } -} - -function updateEvaluators( - evaluators: EvaluatorState, - buildResults: BtBuildResult[], - opts: EvaluatorOpts, -) { - for (const result of buildResults) { - if (result.type === "failure") { - handleBuildFailure({ - result, - terminateOnFailure: opts.terminateOnFailure, - showDetailedErrors: opts.showDetailedErrors, - }); - continue; - } - - for (const evaluator of Object.values(result.evaluator.evaluators)) { - evaluators.evaluators.push({ - sourceFile: result.sourceFile, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - evaluator: evaluator.evaluator as EvaluatorDef< - unknown, - unknown, - unknown, - BaseMetadata - >, - reporter: evaluator.reporter, - }); - } - - for (const [reporterName, reporter] of Object.entries( - result.evaluator.reporters, - )) { - if ( - evaluators.reporters[reporterName] && - evaluators.reporters[reporterName] !== reporter - ) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Reporter '${reporterName}' already exists. Will skip '${reporterName}' from ${result.sourceFile}.`, - ), - ); - continue; - } - evaluators.reporters[reporterName] = reporter; - } - } -} - -async function runAndWatch({ - handles, - onExit, -}: { - handles: Record; - onExit?: () => void; -}) { - const count = Object.keys(handles).length; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Watching ${pluralize("file", count, true)}...`); - - Object.values(handles).map((handle) => handle.watch()); - - ["SIGINT", "SIGTERM"].forEach((signal: string) => { - process.on(signal, function () { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error("Stopped watching."); - for (const handle of Object.values(handles)) { - handle.destroy(); - } - onExit?.(); - process.exit(0); - }); - }); - - // Wait forever while we watch. - await new Promise(() => {}); -} - -export async function buildEvaluators( - handles: Record, - opts: EvaluatorOpts, -): Promise<{ evaluators: EvaluatorState; buildResults: BtBuildResult[] }> { - const buildPromises = Object.values(handles).map((handle) => - handle.rebuild(), - ); - - const buildResults = await Promise.all(buildPromises); - - const evaluators: EvaluatorState = { - evaluators: [], - reporters: {}, - }; - updateEvaluators(evaluators, buildResults, opts); - return { evaluators, buildResults }; -} - -async function runOnce( - handles: Record, - opts: EvaluatorOpts, -) { - const bundlePromises = opts.bundle - ? Object.fromEntries( - Object.entries(handles).map(([inFile, handle]) => [ - inFile, - handle.bundle(), - ]), - ) - : null; - - const { evaluators, buildResults } = await buildEvaluators(handles, opts); - - if (opts.list) { - for (const evaluator of evaluators.evaluators) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log(evaluator.evaluator.evalName); - } - return true; - } - - // map from file name -> eval name -> experiment - const evalToExperiment: Record> = {}; - - const resultPromises = evaluators.evaluators.map(async (evaluator) => { - const evalData = callEvaluatorData(evaluator.evaluator.data); - // TODO: For now, use the eval name as the project. However, we need to evolve - // the definition of a project and create a new concept called run, so that we - // can name the experiment/evaluation within the run the evaluator's name. - const logger = opts.noSendLogs - ? null - : await initExperiment(evaluator.evaluator, evalData); - try { - return await runEvaluator( - logger, - { - ...evaluator.evaluator, - data: evalData.data, - }, - opts.progressReporter, - opts.filters, - undefined, - undefined, - ); - } finally { - if (logger) { - if (!evalToExperiment[evaluator.sourceFile]) { - evalToExperiment[evaluator.sourceFile] = {}; - } - evalToExperiment[evaluator.sourceFile][evaluator.evaluator.evalName] = - logger; - - await logger.flush(); - } - } - }); - - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - styleText( - "dim", - `Processing ${styleText("bold", String(resultPromises.length))} evaluator${resultPromises.length === 1 ? "" : "s"}...`, - ), - ); - const allEvalsResults = await Promise.all(resultPromises); - opts.progressReporter.stop(); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(""); - - const evalReports: Record< - string, - { - reporter: ReporterDef; - results: unknown[]; - } - > = {}; - for (let idx = 0; idx < evaluators.evaluators.length; idx++) { - const evaluator = evaluators.evaluators[idx]; - const resolvedReporter = resolveReporter( - evaluator.reporter, - evaluators.reporters, - ); - - const report = resolvedReporter.reportEval( - evaluator.evaluator, - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - allEvalsResults[idx as number], - { - verbose: opts.showDetailedErrors, - jsonl: opts.jsonl, - }, - ); - - addReport(evalReports, resolvedReporter, report); - } - - if (bundlePromises !== null && Object.entries(evalToExperiment).length > 0) { - await uploadHandleBundles({ - buildResults: buildResults.filter( - // We handle errors above, so it's fine to filter down to successes here. - (result): result is BuildSuccess => result.type === "success", - ), - evalToExperiment, - bundlePromises, - handles, - setCurrent: opts.setCurrent, - defaultIfExists: "replace", - showDetailedErrors: opts.showDetailedErrors, - }); - } - - let allSuccess = true; - for (const [_reporterName, { reporter, results }] of Object.entries( - evalReports, - )) { - const success = await reporter.reportRun(await Promise.all(results)); - allSuccess = allSuccess && success; - } - - return allSuccess; -} - -function checkMatch( - pathInput: string, - include_patterns: string[] | null, - exclude_patterns: string[] | null, -): boolean { - const p = path.resolve(pathInput); - if (include_patterns !== null) { - let include = false; - for (const pattern of include_patterns) { - if (minimatch(p, pattern)) { - include = true; - break; - } - } - if (!include) { - return false; - } - } - - if (exclude_patterns !== null) { - let exclude = false; - for (const pattern of exclude_patterns) { - if (minimatch(p, pattern)) { - exclude = true; - break; - } - } - - return !exclude; - } - - return true; -} - -async function collectFiles( - inputPath: string, - mode: "eval" | "bundle", -): Promise { - let pathStat = null; - try { - pathStat = fs.lstatSync(inputPath); - } catch (e) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Error reading ${inputPath}: ${e}`)); - process.exit(1); - } - - let files: string[] = []; - if (!pathStat.isDirectory()) { - if ( - !checkMatch( - inputPath, - mode === "eval" ? INCLUDE_EVAL : INCLUDE_BUNDLE, - EXCLUDE, - ) - ) { - const prefix = mode === "eval" ? ".eval" : ""; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Reading ${inputPath} because it was specified directly. Rename it to end in ${prefix}.ts or ` + - `${prefix}.js to include it automatically when you specify a directory.`, - ), - ); - } - files.push(inputPath); - } else { - const walked = await util.promisify(fsWalk.walk)(inputPath, { - deepFilter: (entry) => { - return checkMatch(entry.path, null, EXCLUDE); - }, - entryFilter: (entry) => { - return ( - entry.dirent.isFile() && - checkMatch( - entry.path, - mode === "eval" ? INCLUDE_EVAL : INCLUDE_BUNDLE, - EXCLUDE, - ) - ); - }, - }); - - files = files.concat(walked.map((entry) => entry.path)); - } - - return files; -} - -import { createMarkKnownPackagesExternalPlugin } from "./util/external-packages-plugin"; - -// Inspired by and modified from https://github.com/evanw/esbuild/issues/1051 -const nativeNodeModulesPlugin = { - name: "native-node-modules", - setup(build: esbuild.PluginBuild) { - // Keep track of packages that contain .node files - const nativePackages = new Set(); - - // Helper to add a package and its platform-specific variants - const addNativePackage = (pkgName: string) => { - nativePackages.add(pkgName); - if (pkgName.includes("@")) { - const [scope, name] = pkgName.split("/"); - const platformPkgs = [ - `${scope}/${name}-darwin-arm64`, - `${scope}/${name}-darwin-x64`, - `${scope}/${name}-linux-x64-gnu`, - `${scope}/${name}-win32-x64-msvc`, - ]; - platformPkgs.forEach((pkg) => nativePackages.add(pkg)); - } - }; - - // When a .node file is imported, mark its package as native - build.onResolve({ filter: /\.node$/ }, (args) => { - try { - const path = require.resolve(args.path, { paths: [args.resolveDir] }); - const match = path.match( - /node_modules[/\\]((?:@[^/\\]+[/\\])?[^/\\]+)/, - ); - if (match) { - addNativePackage(match[1]); - } - } catch { - // Ignore errors - } - return { path: args.path, external: true }; - }); - - // Handle direct imports of native packages - build.onResolve( - { filter: /@[^/]+\/[^/]+-(?:darwin|linux|win32)/ }, - (args) => { - const match = args.path.match(/^(@[^/]+\/[^/]+)/); - if (match) { - addNativePackage(match[1]); - } - return { path: args.path, external: true }; - }, - ); - - // Mark all imports from native packages as external - build.onResolve({ filter: /.*/ }, (args) => { - if (!args.path.startsWith(".") && !args.path.startsWith("/")) { - const match = args.path.match(/^(?:@[^/]+\/)?[^/]+/); - if (match && nativePackages.has(match[0])) { - return { path: require.resolve(args.path), external: true }; - } - } - return null; - }); - }, -}; - -export type PluginMaker = (fileName: string) => esbuild.Plugin; - -function buildOpts({ - fileName, - outFile, - tsconfig, - plugins: argPlugins, - externalPackages, -}: { - fileName: string; - outFile: string; - tsconfig?: string; - plugins?: PluginMaker[]; - externalPackages?: string[]; -}): esbuild.BuildOptions { - const plugins = [ - braintrustEsbuildPlugin(), - nativeNodeModulesPlugin, - createMarkKnownPackagesExternalPlugin(externalPackages), - ...(argPlugins || []).map((fn) => fn(fileName)), - ]; - return { - entryPoints: [fileName], - bundle: true, - treeShaking: true, - outfile: outFile, - platform: "node", - write: false, - // Remove the leading "v" from process.version - target: `node${process.version.slice(1)}`, - tsconfig, - external: ["node_modules/*", "fsevents"], - plugins: plugins, - }; -} - -export async function initializeHandles({ - files: inputFiles, - mode, - plugins, - tsconfig, - externalPackages, -}: { - files: string[]; - mode: "eval" | "bundle"; - plugins?: PluginMaker[]; - tsconfig?: string; - externalPackages?: string[]; -}): Promise> { - const files: Record = {}; - const inputPaths = inputFiles.length > 0 ? inputFiles : ["."]; - for (const inputPath of inputPaths) { - const newFiles = await collectFiles(inputPath, mode); - if (newFiles.length == 0) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Provided path ${inputPath} is not an eval file or a directory containing eval files, skipping...`, - ), - ); - } - for (const file of newFiles) { - files[path.resolve(file)] = true; - } - } - - if (Object.keys(files).length == 0) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning("No eval files were found in any of the provided paths."), - ); - process.exit(0); - } - - const tmpDir = path.join(os.tmpdir(), `btevals-${uuidv4().slice(0, 8)}`); - // fs.mkdirSync(tmpDir, { recursive: true }); - - const initPromises = []; - for (const file of Object.keys(files)) { - const baseName = `${path.basename( - file, - path.extname(file), - )}-${uuidv4().slice(0, 8)}`; - const outFile = path.join(tmpDir, `${baseName}.${OUT_EXT}`); - const bundleFile = path.join(tmpDir, `${baseName}.bundle.js`); - initPromises.push( - initFile({ - inFile: file, - outFile, - bundleFile, - plugins, - tsconfig, - externalPackages, - }), - ); - } - - const handles: Record = {}; - const initResults = await Promise.all(initPromises); - for (const result of initResults) { - handles[result.inFile] = result; - } - return handles; -} - -async function run(args: RunArgs) { - normalizeDebugLoggingArgs(args); - // Load the environment variables from the .env files using the same rules as Next.js - loadEnvConfig(process.cwd(), true); - - if (args.env_file) { - // Load via dotenv library - const loaded = dotenv.config({ path: args.env_file }); - if (loaded.error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Error loading ${args.env_file}: ${loaded.error}`)); - process.exit(1); - } - } - - const evaluatorOpts: EvaluatorOpts = { - showDetailedErrors: shouldShowDetailedErrors(args.debug_logging), - apiKey: args.api_key, - orgName: args.org_name, - appUrl: args.app_url, - noSendLogs: !!args.no_send_logs, - bundle: !!args.bundle || !!args.push, - setCurrent: !!args.push, - terminateOnFailure: !!args.terminate_on_failure, - watch: !!args.watch, - jsonl: args.jsonl, - progressReporter: args.no_progress_bars - ? new SimpleProgressReporter() - : new BarProgressReporter(), - filters: args.filter ? parseFilters(args.filter) : [], - list: !!args.list, - }; - - if (args.list && args.watch) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error("Cannot specify both --list and --watch.")); - process.exit(1); - } - - const plugins = evaluatorOpts.watch - ? [ - (fileName: string) => - buildWatchPluginForEvaluator(fileName, evaluatorOpts), - ] - : []; - - const handles = await initializeHandles({ - files: args.files, - mode: "eval", - tsconfig: args.tsconfig, - plugins, - externalPackages: args.external_packages, - }); - - if (args.dev) { - // XXX We should watch these files (or support a --watch flag). - const { evaluators } = await buildEvaluators(handles, evaluatorOpts); - const allEvaluators = Object.values(evaluators.evaluators).map( - (e) => e.evaluator, - ); - runDevServer(allEvaluators, { - host: args.dev_host, - port: args.dev_port, - orgName: args.dev_org_name, - }); - return; - } - - let success = true; - try { - if (!evaluatorOpts.noSendLogs) { - await login({ - apiKey: args.api_key, - orgName: args.org_name, - appUrl: args.app_url, - debugLogLevel: args.debug_logging, - }); - } - - if (args.watch) { - await runAndWatch({ - handles, - onExit: () => { - evaluatorOpts.progressReporter.stop(); - }, - }); - } else { - success = await runOnce(handles, evaluatorOpts); - } - } finally { - // ESBuild can freeze up if you do not clean up the handles properly - for (const handle of Object.values(handles)) { - await handle.destroy(); - } - } - - if (!success) { - process.exit(1); - } -} - -function addAuthArgs(parser: ArgumentParser) { - parser.add_argument("--api-key", { - help: "Specify a braintrust api key. If the parameter is not specified, BRAINTRUST_API_KEY or the nearest .env.braintrust file will be used.", - }); - parser.add_argument("--org-name", { - help: "The name of a specific organization to connect to. This is useful if you belong to multiple.", - }); - parser.add_argument("--app-url", { - help: "Specify a custom braintrust app url. Defaults to https://www.braintrust.dev. This is only necessary if you are using an experimental version of Braintrust", - }); - parser.add_argument("--env-file", { - help: "A path to a .env file containing environment variables to load (via dotenv).", - }); -} - -function addDebugLoggingArg(parser: ArgumentParser) { - parser.add_argument("--debug-logging", { - choices: ["error", "warn", "info", "debug"], - help: "Enable internal Braintrust SDK troubleshooting output. Use 'error', 'warn', 'info', or 'debug' to control the log level.", - }); -} - -function addCompileArgs(parser: ArgumentParser) { - parser.add_argument("--terminate-on-failure", { - action: "store_true", - help: "If provided, terminates on a failing eval, instead of the default (moving onto the next one).", - }); - parser.add_argument("--tsconfig", { - help: "Specify a custom tsconfig.json file to use.", - }); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling. These packages will not be included in the bundle and must be available at runtime. Use this to resolve bundling errors with native modules or problematic dependencies. Example: --external-packages sqlite3 fsevents @mapbox/node-pre-gyp", - }); -} - -async function main() { - const parser = new ArgumentParser({ - description: "Braintrust CLI", - }); - - parser.add_argument("-v", "--version", { action: "version", version }); - - const parentParser = new ArgumentParser({ add_help: false }); - parentParser.add_argument("--verbose", { - action: "store_true", - help: "Deprecated alias for --debug-logging debug. Use --debug-logging debug to include full stack traces and detailed troubleshooting output.", - }); - - const subparser = parser.add_subparsers({ - required: true, - }); - - const parser_run = subparser.add_parser("eval", { - help: "Run evals locally.", - parents: [parentParser], - }); - addAuthArgs(parser_run); - addDebugLoggingArg(parser_run); - parser_run.add_argument("--filter", { - help: "Only run evaluators that match these filters. Each filter is a regular expression (https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp). For example, --filter metadata.priority='^P0$' input.name='foo.*bar' will only run evaluators that have metadata.priority equal to 'P0' and input.name matching the regular expression 'foo.*bar'.", - nargs: "*", - }); - parser_run.add_argument("--list", { - help: "List, but do not execute, evaluators.", - action: "store_true", - }); - parser_run.add_argument("--jsonl", { - action: "store_true", - help: "Format score summaries as jsonl, i.e. one JSON-formatted line per summary.", - }); - addCompileArgs(parser_run); - parser_run.add_argument("--watch", { - action: "store_true", - help: "Watch files for changes and rerun evals when changes are detected", - }); - parser_run.add_argument("--no-send-logs", { - action: "store_true", - help: "Do not send logs to Braintrust. Useful for testing evaluators without uploading results.", - }); - parser_run.add_argument("--no-progress-bars", { - action: "store_true", - help: "Do not show progress bars when processing evaluators.", - }); - parser_run.add_argument("--bundle", { - action: "store_true", - help: "Experimental (do not use unless you know what you're doing)", - }); - parser_run.add_argument("--push", { - action: "store_true", - help: "Push the scorers from the current run to Braintrust. This will mark the current run's scorers as the latest version in the project.", - }); - parser_run.add_argument("files", { - nargs: "*", - help: "A list of files or directories to run. If no files are specified, the current directory is used.", - }); - parser_run.add_argument("--dev", { - action: "store_true", - help: "Run the evaluators in dev mode. This will start a dev server which you can connect to via the playground's remote evals feature.", - }); - parser_run.add_argument("--dev-host", { - help: "The host to bind the dev server to. Defaults to localhost. Set to 0.0.0.0 to bind to all interfaces.", - type: String, - default: "localhost", - }); - parser_run.add_argument("--dev-port", { - help: "The port to bind the dev server to. Defaults to 8300.", - type: Number, - default: 8300, - }); - parser_run.add_argument("--dev-org-name", { - help: "Only allow users that belong this to this org name to run remote evals.", - type: String, - }); - parser_run.set_defaults({ func: run }); - - const parser_push = subparser.add_parser("push", { - help: "Bundle prompts, tools, scorers, and other resources into Braintrust", - }); - addAuthArgs(parser_push); - addDebugLoggingArg(parser_push); - addCompileArgs(parser_push); - parser_push.add_argument("files", { - nargs: "*", - help: "A list of files or directories containing functions to bundle. If no files are specified, the current directory is used.", - }); - parser_push.add_argument("--if-exists", { - choices: ["error", "replace", "ignore"], - default: "error", - help: "What to do if a function with the same slug already exists. 'error' will cause an error and abort. 'replace' will overwrite the existing function. 'ignore' will ignore the push for this function and continue.", - }); - parser_push.set_defaults({ func: bundleCommand }); - - const parser_pull = subparser.add_parser("pull", { - help: "Pull prompts, tools, scorers, and other resources from Braintrust to save in your codebase.", - }); - addDebugLoggingArg(parser_pull); - parser_pull.add_argument("--output-dir", { - help: "The directory to output the pulled resources to. If not specified, the current directory is used.", - }); - parser_pull.add_argument("--project-name", { - help: "The name of the project to pull from. If not specified, all projects are pulled.", - }); - parser_pull.add_argument("--project-id", { - help: "The id of the project to pull from. If not specified, all projects are pulled.", - }); - parser_pull.add_argument("--id", { - help: "The id of a specific function to pull.", - }); - parser_pull.add_argument("--slug", { - help: "The slug of a specific function to pull.", - }); - parser_pull.add_argument("--version", { - help: "The version to pull. Will pull the latest version of each prompt that is at or before this version.", - }); - parser_pull.add_argument("--force", { - action: "store_true", - help: "Overwrite local files if they have uncommitted changes.", - }); - parser_pull.set_defaults({ func: pullCommand }); - - const parsed = normalizeDebugLoggingArgs(parser.parse_args()); - - try { - await parsed.func(parsed); - } catch (e) { - logError(e, shouldShowDetailedErrors(parsed.debug_logging)); - process.exit(1); - } -} - -if (require.main === module) { - void main(); -} diff --git a/js/src/cli/integration.test.ts b/js/src/cli/integration.test.ts deleted file mode 100644 index ae20ec41f..000000000 --- a/js/src/cli/integration.test.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { CompileArgs, RunArgs, BundleArgs, PullArgs } from "./util/types"; - -// Test the type definitions for external packages -describe("External Packages Type Definitions", () => { - describe("CompileArgs interface", () => { - it("should have external_packages property", () => { - const args: CompileArgs = { - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: ["sqlite3", "fsevents"], - }; - - expect(args.external_packages).toEqual(["sqlite3", "fsevents"]); - }); - - it("should allow undefined external_packages", () => { - const args: CompileArgs = { - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: undefined, - }; - - expect(args.external_packages).toBeUndefined(); - }); - - it("should allow empty external_packages array", () => { - const args: CompileArgs = { - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: [], - }; - - expect(args.external_packages).toEqual([]); - }); - }); - - describe("RunArgs interface", () => { - it("should inherit external_packages from CompileArgs", () => { - const args: RunArgs = { - verbose: false, - api_key: "test-key", - org_name: "test-org", - app_url: "https://test.com", - env_file: ".env", - debug_logging: "info", - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: ["@mapbox/node-pre-gyp"], - files: ["test.eval.ts"], - watch: false, - list: false, - jsonl: false, - filter: undefined, - no_send_logs: false, - no_progress_bars: false, - bundle: false, - push: false, - dev: false, - dev_host: "localhost", - dev_port: 8300, - }; - - expect(args.external_packages).toEqual(["@mapbox/node-pre-gyp"]); - expect(args.debug_logging).toBe("info"); - }); - }); - - describe("BundleArgs interface", () => { - it("should inherit external_packages from CompileArgs", () => { - const args: BundleArgs = { - verbose: false, - api_key: "test-key", - org_name: "test-org", - app_url: "https://test.com", - env_file: ".env", - debug_logging: "debug", - tsconfig: "tsconfig.json", - terminate_on_failure: false, - external_packages: ["sqlite3", "sharp"], - files: ["functions.ts"], - if_exists: "error", - }; - - expect(args.external_packages).toEqual(["sqlite3", "sharp"]); - expect(args.debug_logging).toBe("debug"); - }); - }); - - describe("PullArgs interface", () => { - it("should include debug_logging from CommonArgs", () => { - const args: PullArgs = { - verbose: false, - output_dir: "./braintrust", - project_name: "test-project", - force: false, - debug_logging: "info", - }; - - expect(args.debug_logging).toBe("info"); - }); - }); -}); - -// Test the buildOpts function parameter flow -describe("buildOpts Function Integration", () => { - // This is a simplified version of the buildOpts function signature - interface BuildOptsParams { - fileName: string; - outFile: string; - tsconfig?: string; - plugins?: any[]; - externalPackages?: string[]; - } - - it("should accept externalPackages parameter", () => { - const params: BuildOptsParams = { - fileName: "test.ts", - outFile: "test.js", - tsconfig: "tsconfig.json", - plugins: [], - externalPackages: ["sqlite3", "fsevents"], - }; - - expect(params.externalPackages).toEqual(["sqlite3", "fsevents"]); - }); - - it("should handle undefined externalPackages", () => { - const params: BuildOptsParams = { - fileName: "test.ts", - outFile: "test.js", - tsconfig: "tsconfig.json", - plugins: [], - externalPackages: undefined, - }; - - expect(params.externalPackages).toBeUndefined(); - }); - - it("should handle empty externalPackages array", () => { - const params: BuildOptsParams = { - fileName: "test.ts", - outFile: "test.js", - tsconfig: "tsconfig.json", - plugins: [], - externalPackages: [], - }; - - expect(params.externalPackages).toEqual([]); - }); -}); - -// Test the parameter flow from CLI to buildOpts -describe("Parameter Flow Integration", () => { - it("should flow external_packages from RunArgs to buildOpts", () => { - // Simulate the parameter flow: RunArgs -> initializeHandles -> initFile -> buildOpts - const runArgs: Partial = { - external_packages: ["sqlite3", "@mapbox/node-pre-gyp"], - }; - - // This would be passed to initializeHandles - const initializeHandlesParams = { - files: ["test.eval.ts"], - mode: "eval" as const, - tsconfig: undefined, - plugins: undefined, - externalPackages: runArgs.external_packages, - }; - - expect(initializeHandlesParams.externalPackages).toEqual([ - "sqlite3", - "@mapbox/node-pre-gyp", - ]); - - // This would be passed to initFile - const initFileParams = { - inFile: "test.eval.ts", - outFile: "test.js", - bundleFile: "test.bundle.js", - tsconfig: undefined, - plugins: undefined, - externalPackages: initializeHandlesParams.externalPackages, - }; - - expect(initFileParams.externalPackages).toEqual([ - "sqlite3", - "@mapbox/node-pre-gyp", - ]); - - // This would be passed to buildOpts - const buildOptsParams = { - fileName: initFileParams.inFile, - outFile: initFileParams.outFile, - tsconfig: initFileParams.tsconfig, - plugins: initFileParams.plugins, - externalPackages: initFileParams.externalPackages, - }; - - expect(buildOptsParams.externalPackages).toEqual([ - "sqlite3", - "@mapbox/node-pre-gyp", - ]); - }); - - it("should flow external_packages from BundleArgs to buildOpts", () => { - // Simulate the parameter flow: BundleArgs -> initializeHandles -> initFile -> buildOpts - const bundleArgs: Partial = { - external_packages: ["sharp", "canvas"], - }; - - // This would be passed to initializeHandles - const initializeHandlesParams = { - files: ["functions.ts"], - mode: "bundle" as const, - tsconfig: undefined, - plugins: undefined, - externalPackages: bundleArgs.external_packages, - }; - - expect(initializeHandlesParams.externalPackages).toEqual([ - "sharp", - "canvas", - ]); - - // This would be passed to buildOpts - const buildOptsParams = { - fileName: "functions.ts", - outFile: "functions.js", - tsconfig: undefined, - plugins: undefined, - externalPackages: initializeHandlesParams.externalPackages, - }; - - expect(buildOptsParams.externalPackages).toEqual(["sharp", "canvas"]); - }); -}); - -// Test real-world usage scenarios -describe("Real-world Usage Scenarios", () => { - it("should handle autoevals dependency issue scenario", () => { - const args: Partial = { - files: ["autoevals-test.eval.ts"], - external_packages: ["@mapbox/node-pre-gyp", "mock-aws-s3", "aws-sdk"], - }; - - // These packages would be added to the external list - const expectedExternals = [ - // Hardcoded externals - "braintrust", - "autoevals", - "@braintrust/", - "config", - "lightningcss", - "@mapbox/node-pre-gyp", - // CLI-specified externals - ...args.external_packages!, - ]; - - expect(expectedExternals).toContain("@mapbox/node-pre-gyp"); - expect(expectedExternals).toContain("mock-aws-s3"); - expect(expectedExternals).toContain("aws-sdk"); - }); - - it("should handle database packages scenario", () => { - const args: Partial = { - files: ["database-test.eval.ts"], - external_packages: ["sqlite3", "better-sqlite3", "mysql2"], - }; - - expect(args.external_packages).toEqual([ - "sqlite3", - "better-sqlite3", - "mysql2", - ]); - }); - - it("should handle native modules scenario", () => { - const args: Partial = { - files: ["image-processing.eval.ts"], - external_packages: ["sharp", "canvas", "node-sass"], - }; - - expect(args.external_packages).toEqual(["sharp", "canvas", "node-sass"]); - }); -}); diff --git a/js/src/cli/jest/nodeModulesPaths.ts b/js/src/cli/jest/nodeModulesPaths.ts deleted file mode 100644 index f04748e9a..000000000 --- a/js/src/cli/jest/nodeModulesPaths.ts +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - * - * Adapted from: https://github.com/substack/node-resolve - */ - -import * as path from "node:path"; -import { realpathSync } from "node:fs"; - -type NodeModulesPathsOptions = { - moduleDirectory?: Array; - paths?: Array; -}; - -export default function nodeModulesPaths( - basedir: string, - options: NodeModulesPathsOptions, -): Array { - const modules = - options && options.moduleDirectory - ? Array.from(options.moduleDirectory) - : ["node_modules"]; - - // ensure that `basedir` is an absolute path at this point, - // resolving against the process' current working directory - const basedirAbs = path.resolve(basedir); - - let prefix = "/"; - if (/^([A-Za-z]:)/.test(basedirAbs)) { - prefix = ""; - } else if (/^\\\\/.test(basedirAbs)) { - prefix = "\\\\"; - } - - // The node resolution algorithm (as implemented by NodeJS and TypeScript) - // traverses parents of the physical path, not the symlinked path - let physicalBasedir; - try { - physicalBasedir = realpathSync.native(basedirAbs); - } catch { - // realpath can throw, e.g. on mapped drives - physicalBasedir = basedirAbs; - } - - const paths: Array = [physicalBasedir]; - let parsed = path.parse(physicalBasedir); - while (parsed.dir !== paths[paths.length - 1]) { - paths.push(parsed.dir); - parsed = path.parse(parsed.dir); - } - - const dirs = paths.reduce>((dirs, aPath) => { - for (const moduleDir of modules) { - if (path.isAbsolute(moduleDir)) { - if (aPath === basedirAbs && moduleDir) { - dirs.push(moduleDir); - } - } else { - dirs.push(path.join(prefix, aPath, moduleDir)); - } - } - - return dirs; - }, []); - - if (options.paths) { - dirs.push(...options.paths); - } - - return dirs; -} diff --git a/js/src/cli/progress.ts b/js/src/cli/progress.ts deleted file mode 100644 index befab92f6..000000000 --- a/js/src/cli/progress.ts +++ /dev/null @@ -1 +0,0 @@ -export { BarProgressReporter } from "./reporters/progress"; diff --git a/js/src/cli/reporters/eval.ts b/js/src/cli/reporters/eval.ts deleted file mode 100644 index 8f7ff8708..000000000 --- a/js/src/cli/reporters/eval.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { stripVTControlCharacters, styleText } from "node:util"; -import { terminalLink } from "termi-link"; -import Table from "cli-table3"; -import pluralize from "pluralize"; - -import { ExperimentSummary, ScoreSummary, MetricSummary } from "../../logger"; -import type { ReporterDef } from "../../reporters/types"; -import { EvaluatorDef, EvalResultWithSummary } from "../../framework"; -import { isEmpty } from "../../util"; - -function visibleLength(text: string) { - return stripVTControlCharacters(text).length; -} - -function padEndVisible(text: string, targetLength: number) { - return text + " ".repeat(Math.max(0, targetLength - visibleLength(text))); -} - -function formatSummaryBox(content: string) { - const title = styleText("gray", " Experiment summary "); - const lines = content.split("\n"); - const contentWidth = Math.max( - visibleLength(title), - ...lines.map((line) => visibleLength(line) + 2), - ); - - const horizontal = "─"; - const top = - styleText("gray", "╭") + - title + - styleText( - "gray", - horizontal.repeat(contentWidth - visibleLength(title)) + "╮", - ); - const body = lines - .map( - (line) => - `${styleText("gray", "│")} ${padEndVisible(line, contentWidth - 2)} ${styleText("gray", "│")}`, - ) - .join("\n"); - const bottom = styleText("gray", "╰" + horizontal.repeat(contentWidth) + "╯"); - - return top + "\n" + body + "\n" + bottom; -} - -function formatExperimentSummaryFancy(summary: ExperimentSummary) { - let comparisonLine = ""; - if (summary.comparisonExperimentName) { - comparisonLine = `${summary.comparisonExperimentName} ${styleText("gray", "(baseline)")} ← ${summary.experimentName} ${styleText("gray", "(comparison)")}\n\n`; - } - - const tableParts: string[] = []; - - const hasScores = Object.keys(summary.scores).length > 0; - const hasMetrics = Object.keys(summary.metrics ?? {}).length > 0; - const hasComparison = !!summary.comparisonExperimentName; - - if (hasScores || hasMetrics) { - const headers = [styleText("gray", "Name"), styleText("gray", "Value")]; - - if (hasComparison) { - headers.push( - styleText("gray", "Change"), - styleText("gray", "Improvements"), - styleText("gray", "Regressions"), - ); - } - - const combinedTable = new Table({ - head: hasComparison ? headers : [], - style: { head: [], "padding-left": 0, "padding-right": 0, border: [] }, - chars: { - top: "", - "top-mid": "", - "top-left": "", - "top-right": "", - bottom: "", - "bottom-mid": "", - "bottom-left": "", - "bottom-right": "", - left: "", - "left-mid": "", - mid: "", - "mid-mid": "", - right: "", - "right-mid": "", - middle: " ", - }, - colWidths: hasComparison ? [18, 10, 10, 13, 12] : [20, 15], - colAligns: hasComparison - ? ["left", "right", "right", "right", "right"] - : ["left", "right"], - wordWrap: false, - }); - - const scoreValues: ScoreSummary[] = Object.values(summary.scores); - for (const score of scoreValues) { - const scorePercent = (score.score * 100).toFixed(2); - const scoreValue = styleText("white", `${scorePercent}%`); - - let diffString = ""; - if (!isEmpty(score.diff)) { - const diffPercent = (score.diff! * 100).toFixed(2); - const diffSign = score.diff! > 0 ? "+" : ""; - const diffColor = score.diff! > 0 ? "green" : "red"; - diffString = styleText(diffColor, `${diffSign}${diffPercent}%`); - } else { - diffString = styleText("gray", "-"); - } - - const improvements = - score.improvements > 0 - ? styleText(["dim", "green"], String(score.improvements)) - : styleText("gray", "-"); - const regressions = - score.regressions > 0 - ? styleText(["dim", "red"], String(score.regressions)) - : styleText("gray", "-"); - - const row = [`${styleText("blue", "◯")} ${score.name}`, scoreValue]; - if (hasComparison) { - row.push(diffString, improvements, regressions); - } - combinedTable.push(row); - } - - const metricValues: MetricSummary[] = Object.values(summary.metrics ?? {}); - for (const metric of metricValues) { - const fractionDigits = Number.isInteger(metric.metric) ? 0 : 2; - const formattedValue = metric.metric.toFixed(fractionDigits); - const metricValue = styleText( - "white", - metric.unit === "$" - ? `${metric.unit}${formattedValue}` - : `${formattedValue}${metric.unit}`, - ); - - let diffString = ""; - if (!isEmpty(metric.diff)) { - const diffPercent = (metric.diff! * 100).toFixed(2); - const diffSign = metric.diff! > 0 ? "+" : ""; - const diffColor = metric.diff! > 0 ? "green" : "red"; - diffString = styleText(diffColor, `${diffSign}${diffPercent}%`); - } else { - diffString = styleText("gray", "-"); - } - - const improvements = - metric.improvements > 0 - ? styleText(["dim", "green"], String(metric.improvements)) - : styleText("gray", "-"); - const regressions = - metric.regressions > 0 - ? styleText(["dim", "red"], String(metric.regressions)) - : styleText("gray", "-"); - - const row = [`${styleText("magenta", "◯")} ${metric.name}`, metricValue]; - if (hasComparison) { - row.push(diffString, improvements, regressions); - } - combinedTable.push(row); - } - - tableParts.push(combinedTable.toString()); - } - - const content = [comparisonLine, ...tableParts].filter(Boolean).join("\n"); - - const footer = summary.experimentUrl - ? terminalLink( - `View results for ${summary.experimentName}`, - summary.experimentUrl, - { fallback: () => `See results at ${summary.experimentUrl}` }, - ) - : ""; - - const boxContent = [content, footer].filter(Boolean).join("\n\n"); - - return "\n" + formatSummaryBox(boxContent); -} - -export const warning = (text: string) => styleText("yellow", text); - -export const fancyReporter: ReporterDef = { - name: "Braintrust fancy reporter", - async reportEval( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - evaluator: EvaluatorDef, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - result: EvalResultWithSummary, - { verbose, jsonl }: { verbose: boolean; jsonl?: boolean }, - ) { - const { results, summary } = result; - const failingResults = results.filter( - (r: { error: unknown }) => r.error !== undefined, - ); - - if (failingResults.length > 0) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - warning( - `Evaluator ${evaluator.evalName} failed with ${pluralize("error", failingResults.length, true)}. This evaluation ("${evaluator.evalName}") will not be fully logged.`, - ), - ); - if (jsonl) { - for (const result of failingResults) { - process.stdout.write(JSON.stringify(result)); - process.stdout.write("\n"); - } - } else if (verbose) { - for (const result of failingResults) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(result); - } - } - } - - process.stdout.write( - jsonl ? JSON.stringify(summary) : formatExperimentSummaryFancy(summary), - ); - process.stdout.write("\n"); - return failingResults.length === 0; - }, - async reportRun(evalReports: boolean[]) { - return evalReports.every((r) => r); - }, -}; diff --git a/js/src/cli/reporters/progress.ts b/js/src/cli/reporters/progress.ts deleted file mode 100644 index e764709f3..000000000 --- a/js/src/cli/reporters/progress.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { styleText } from "node:util"; -import * as cliProgress from "cli-progress"; - -import type { ProgressReporter } from "../../reporters/types"; - -export { SimpleProgressReporter } from "../../reporters/progress"; - -const MAX_NAME_LENGTH = 40; - -function fitNameToSpaces(name: string, length: number) { - const padded = name.padEnd(length); - if (padded.length <= length) { - return padded; - } - return padded.substring(0, length - 3) + "..."; -} - -export class BarProgressReporter implements ProgressReporter { - private multiBar: cliProgress.MultiBar; - private bars: Record = {}; - - constructor() { - this.multiBar = new cliProgress.MultiBar( - { - format: `${styleText("blueBright", "{bar}")} ${styleText("blue", "{evaluator}")} {percentage}% ${styleText("gray", "{value}/{total} {eta_formatted}")}`, - hideCursor: true, - barsize: 10, - }, - cliProgress.Presets.shades_grey, - ); - } - - public start(name: string, total: number) { - const bar = this.multiBar.create(total, 0); - this.bars[name] = bar; - } - - public stop() { - this.multiBar.stop(); - } - - public increment(name: string) { - this.bars[name].increment({ - evaluator: fitNameToSpaces(name, MAX_NAME_LENGTH), - }); - } - - public setTotal(name: string, total: number) { - this.bars[name].setTotal(total); - } -} diff --git a/js/src/cli/types.ts b/js/src/cli/types.ts deleted file mode 100644 index 15184a247..000000000 --- a/js/src/cli/types.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type * as esbuild from "esbuild"; -import type { BaseMetadata } from "../logger"; -import type { EvaluatorDef, EvaluatorFile } from "../framework"; -import type { ReporterDef } from "../reporters/types"; - -export interface BuildSuccess { - type: "success"; - result: esbuild.BuildResult; - evaluator: EvaluatorFile; - sourceFile: string; -} - -export interface BuildFailure { - type: "failure"; - error: Error; - sourceFile: string; -} - -export type BtBuildResult = BuildSuccess | BuildFailure; - -export interface FileHandle { - inFile: string; - outFile: string; - bundleFile?: string; - rebuild: () => Promise; - bundle: () => Promise; - watch: () => void; - destroy: () => Promise; -} - -export interface EvaluatorState { - evaluators: { - sourceFile: string; - evaluator: EvaluatorDef; - reporter: string | ReporterDef | undefined; - }[]; - reporters: { - [reporter: string]: ReporterDef; - }; -} diff --git a/js/src/cli/util/bundle.ts b/js/src/cli/util/bundle.ts deleted file mode 100644 index 9eb18bced..000000000 --- a/js/src/cli/util/bundle.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { loadEnvConfig } from "@next/env"; -import * as dotenv from "dotenv"; -import { AuthArgs, BundleArgs, CommonArgs } from "./types"; -import { error } from "../../framework"; -import type { BtBuildResult } from "../types"; -import { handleBuildFailure, initializeHandles } from "../index"; -import { login } from "../../logger"; -import { uploadHandleBundles } from "../functions/upload"; -import { - normalizeDebugLoggingArgs, - shouldShowDetailedErrors, -} from "./debug-logging"; - -export async function loadCLIEnv(args: AuthArgs & CommonArgs) { - normalizeDebugLoggingArgs(args); - // Load the environment variables from the .env files using the same rules as Next.js - loadEnvConfig(process.cwd(), true); - - if (args.env_file) { - // Load via dotenv library - const loaded = dotenv.config({ path: args.env_file }); - if (loaded.error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(error(`Error loading ${args.env_file}: ${loaded.error}`)); - process.exit(1); - } - } - - await login({ - apiKey: args.api_key, - orgName: args.org_name, - appUrl: args.app_url, - debugLogLevel: args.debug_logging, - }); -} - -export async function bundleCommand(args: BundleArgs) { - await loadCLIEnv(args); - const showDetailedErrors = shouldShowDetailedErrors(args.debug_logging); - - const handles = await initializeHandles({ - mode: "bundle", - files: args.files, - tsconfig: args.tsconfig, - externalPackages: args.external_packages, - }); - - try { - const allBuildResultsP: Promise[] = Object.values( - handles, - ).map((handle) => handle.rebuild()); - - const bundlePromises = Object.fromEntries( - Object.entries(handles).map(([inFile, handle]) => [ - inFile, - handle.bundle(), - ]), - ); - - const allBuildResults = await Promise.all(allBuildResultsP); - const buildResults = []; - for (const buildResult of allBuildResults) { - if (buildResult.type === "failure") { - handleBuildFailure({ - result: buildResult, - terminateOnFailure: args.terminate_on_failure, - showDetailedErrors, - }); - } else { - buildResults.push(buildResult); - } - } - - const { numFailed } = await uploadHandleBundles({ - buildResults, - bundlePromises, - handles, - setCurrent: true, - showDetailedErrors, - defaultIfExists: args.if_exists, - }); - - if (numFailed > 0) { - process.exit(1); - } - } finally { - for (const handle of Object.values(handles)) { - await handle.destroy(); - } - } -} diff --git a/js/src/cli/util/debug-logging.test.ts b/js/src/cli/util/debug-logging.test.ts deleted file mode 100644 index 320911e5e..000000000 --- a/js/src/cli/util/debug-logging.test.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { beforeEach, describe, expect, test, vi } from "vitest"; - -import { - normalizeDebugLoggingArgs, - resetDebugLoggingArgsForTests, - shouldShowDetailedErrors, - VERBOSE_DEPRECATION_MESSAGE, -} from "./debug-logging"; - -describe("CLI debug logging helpers", () => { - beforeEach(() => { - vi.restoreAllMocks(); - resetDebugLoggingArgsForTests(); - }); - - test("shouldShowDetailedErrors only enables detailed errors for debug", () => { - expect(shouldShowDetailedErrors(undefined)).toBe(false); - expect(shouldShowDetailedErrors("info")).toBe(false); - expect(shouldShowDetailedErrors("debug")).toBe(true); - }); - - test("normalizeDebugLoggingArgs upgrades verbose to debug and warns once", () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - const args = { - verbose: true, - debug_logging: undefined as - | "error" - | "warn" - | "info" - | "debug" - | undefined, - }; - - normalizeDebugLoggingArgs(args); - normalizeDebugLoggingArgs(args); - - expect(args.debug_logging).toBe("debug"); - expect(warnSpy).toHaveBeenCalledTimes(1); - expect(warnSpy).toHaveBeenCalledWith( - `Warning: ${VERBOSE_DEPRECATION_MESSAGE}`, - ); - }); - - test("normalizeDebugLoggingArgs preserves explicit debug logging", () => { - const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - const args = { verbose: true, debug_logging: "info" as const }; - - normalizeDebugLoggingArgs(args); - - expect(args.debug_logging).toBe("info"); - expect(warnSpy).toHaveBeenCalledTimes(1); - }); -}); diff --git a/js/src/cli/util/debug-logging.ts b/js/src/cli/util/debug-logging.ts deleted file mode 100644 index 544514a11..000000000 --- a/js/src/cli/util/debug-logging.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { warning } from "../../framework"; -import type { CommonArgs } from "./types"; - -export const VERBOSE_DEPRECATION_MESSAGE = - "--verbose is deprecated and will be removed in a future version of braintrust. Use --debug-logging debug to see full stack traces and troubleshooting details."; - -let hasWarnedAboutVerboseFlag = false; - -export function shouldShowDetailedErrors( - debugLogLevel: CommonArgs["debug_logging"] | undefined, -): boolean { - return debugLogLevel === "debug"; -} - -export function normalizeDebugLoggingArgs< - T extends Pick, ->(args: T): T { - if (!args.verbose) { - return args; - } - - if (!hasWarnedAboutVerboseFlag) { - hasWarnedAboutVerboseFlag = true; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn(warning(VERBOSE_DEPRECATION_MESSAGE)); - } - - if (!args.debug_logging) { - args.debug_logging = "debug"; - } - - return args; -} - -export function resetDebugLoggingArgsForTests(): void { - hasWarnedAboutVerboseFlag = false; -} diff --git a/js/src/cli/util/external-packages-plugin.test.ts b/js/src/cli/util/external-packages-plugin.test.ts deleted file mode 100644 index 34011d05f..000000000 --- a/js/src/cli/util/external-packages-plugin.test.ts +++ /dev/null @@ -1,319 +0,0 @@ -import { describe, it, expect, vi } from "vitest"; -import { createMarkKnownPackagesExternalPlugin } from "./external-packages-plugin"; - -describe("External Packages Plugin", () => { - describe("createMarkKnownPackagesExternalPlugin", () => { - it("should create a plugin with the correct name", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - expect(plugin.name).toBe("make-known-packages-external"); - }); - - it("should mark hardcoded packages as external", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - expect(mockBuild.onResolve).toHaveBeenCalledWith( - { filter: expect.any(RegExp) }, - expect.any(Function), - ); - - // Get the filter regex that was passed - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - const resolveFunction = mockBuild.onResolve.mock.calls[0][1]; - - // Test hardcoded packages - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("autoevals")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - expect(filter.test("config")).toBe(true); - expect(filter.test("lightningcss")).toBe(true); - - // Test that the resolve function returns external: true - expect(resolveFunction({ path: "braintrust" })).toEqual({ - path: "braintrust", - external: true, - }); - }); - - it("should not mark non-matching packages as external", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test packages that should NOT match - expect(filter.test("react")).toBe(false); - expect(filter.test("lodash")).toBe(false); - expect(filter.test("@types/node")).toBe(false); - expect(filter.test("some-random-package")).toBe(false); - }); - - it("should include additional packages from CLI", () => { - const additionalPackages = ["sqlite3", "fsevents", "@scope/package"]; - const plugin = createMarkKnownPackagesExternalPlugin(additionalPackages); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test that additional packages are included - expect(filter.test("sqlite3")).toBe(true); - expect(filter.test("fsevents")).toBe(true); - expect(filter.test("@scope/package")).toBe(true); - - // Test that hardcoded packages still work - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - }); - - it("should handle packages with subpaths", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test subpaths - expect(filter.test("braintrust/core")).toBe(true); - expect(filter.test("braintrust/dist/index.js")).toBe(true); - expect( - filter.test("@mapbox/node-pre-gyp/lib/util/nw-pre-gyp/index.html"), - ).toBe(true); - }); - - it("should handle special characters in package names", () => { - const additionalPackages = [ - "@scope/package-with-dashes", - "package.with.dots", - ]; - const plugin = createMarkKnownPackagesExternalPlugin(additionalPackages); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test packages with special characters - expect(filter.test("@scope/package-with-dashes")).toBe(true); - expect(filter.test("package.with.dots")).toBe(true); - }); - - it("should resolve the original @mapbox/node-pre-gyp bundling issue", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - const resolveFunction = mockBuild.onResolve.mock.calls[0][1]; - - // Test the specific problematic paths from the original issue - const problematicPaths = [ - "@mapbox/node-pre-gyp", - "@mapbox/node-pre-gyp/lib/util/nw-pre-gyp/index.html", - "@mapbox/node-pre-gyp/lib/util/s3_setup.js", - "@mapbox/node-pre-gyp/lib/node-pre-gyp.js", - ]; - - problematicPaths.forEach((path) => { - expect(filter.test(path)).toBe(true); - expect(resolveFunction({ path })).toEqual({ - path, - external: true, - }); - }); - }); - - it("should prevent false positives with similar package names", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test cases that should NOT match (false positives) - const falsePositives = [ - "braintrust-extended", - "my-braintrust", - "config-loader", - "lightningcss-plugin", - "sqlite3-wrapper", - "better-sqlite3", - "node-pre-gyp", // Similar to @mapbox/node-pre-gyp but different - "@other/braintrust", - ]; - - falsePositives.forEach((packageName) => { - expect(filter.test(packageName)).toBe(false); - }); - - // Test cases that SHOULD match (true positives) - const truePositives = [ - "braintrust", - "braintrust/core", - "braintrust/dist/index.js", - "@mapbox/node-pre-gyp", - "@mapbox/node-pre-gyp/lib/util/nw-pre-gyp/index.html", - "config", - "lightningcss", - ]; - - truePositives.forEach((packageName) => { - expect(filter.test(packageName)).toBe(true); - }); - }); - - it("should handle prefix matching vs exact matching correctly", () => { - const plugin = createMarkKnownPackagesExternalPlugin(); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Test prefix matching for @braintrust/ (ends with /) - const braintrustPrefixTests = [ - { - input: "@braintrust/utils", - expected: true, - description: "Should match @braintrust/ prefix", - }, - { - input: "@braintrust-other/core", - expected: false, - description: "Should not match similar but different scope", - }, - ]; - - // Test exact matching for config (doesn't end with /) - const configExactTests = [ - { - input: "config", - expected: true, - description: "Should match config exactly", - }, - { - input: "config/local", - expected: true, - description: "Should match config with subpath", - }, - { - input: "config-loader", - expected: false, - description: "Should not match config with suffix", - }, - { - input: "my-config", - expected: false, - description: "Should not match config with prefix", - }, - ]; - - // Test exact matching for braintrust (doesn't end with /) - const braintrustExactTests = [ - { - input: "braintrust", - expected: true, - description: "Should match braintrust exactly", - }, - { - input: "braintrust/core", - expected: true, - description: "Should match braintrust with subpath", - }, - { - input: "braintrust-extended", - expected: false, - description: "Should not match braintrust with suffix", - }, - { - input: "my-braintrust", - expected: false, - description: "Should not match braintrust with prefix", - }, - ]; - - const allTests = [ - ...braintrustPrefixTests, - ...configExactTests, - ...braintrustExactTests, - ]; - - allTests.forEach((test) => { - const result = filter.test(test.input); - expect(result).toBe(test.expected); - }); - }); - }); - - describe("Regex Edge Cases", () => { - it("should handle empty additional packages array", () => { - const plugin = createMarkKnownPackagesExternalPlugin([]); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Should still work with hardcoded packages - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - }); - - it("should handle undefined additional packages", () => { - const plugin = createMarkKnownPackagesExternalPlugin(undefined); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Should still work with hardcoded packages - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("@mapbox/node-pre-gyp")).toBe(true); - }); - - it("should handle duplicate packages", () => { - const additionalPackages = ["braintrust", "sqlite3", "braintrust"]; // duplicate - const plugin = createMarkKnownPackagesExternalPlugin(additionalPackages); - const mockBuild = { - onResolve: vi.fn(), - }; - - plugin.setup(mockBuild as any); - - const [{ filter }] = mockBuild.onResolve.mock.calls[0]; - - // Should still work correctly - expect(filter.test("braintrust")).toBe(true); - expect(filter.test("sqlite3")).toBe(true); - }); - }); -}); diff --git a/js/src/cli/util/external-packages-plugin.ts b/js/src/cli/util/external-packages-plugin.ts deleted file mode 100644 index 115a98bff..000000000 --- a/js/src/cli/util/external-packages-plugin.ts +++ /dev/null @@ -1,43 +0,0 @@ -import * as esbuild from "esbuild"; - -// Inspired by https://github.com/evanw/esbuild/issues/619 -// In addition to marking node_modules external, explicitly mark -// our packages (braintrust and autoevals) external, in case they're -// installed in a relative path. -export function createMarkKnownPackagesExternalPlugin( - additionalPackages: string[] = [], -) { - return { - name: "make-known-packages-external", - setup(build: esbuild.PluginBuild) { - // Mark known packages as external - const knownPackages = [ - "braintrust", - "autoevals", - "@braintrust/", - "config", - "lightningcss", - "@mapbox/node-pre-gyp", - "fsevents", - "chokidar", - ...additionalPackages, - ]; - const escapedPackages = knownPackages.map((pkg) => { - const escaped = pkg.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); - // For packages ending with /, match anything with that prefix - if (pkg.endsWith("/")) { - return escaped + ".*"; - } - // For regular packages, match exact name or name followed by / - return escaped + "(?:\\/.*)?"; - }); - const knownPackagesFilter = new RegExp( - `^(${escapedPackages.join("|")})$`, - ); - build.onResolve({ filter: knownPackagesFilter }, (args) => ({ - path: args.path, - external: true, - })); - }, - }; -} diff --git a/js/src/cli/util/external-packages.test.ts b/js/src/cli/util/external-packages.test.ts deleted file mode 100644 index 210279917..000000000 --- a/js/src/cli/util/external-packages.test.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { describe, it, expect } from "vitest"; -import { ArgumentParser } from "argparse"; - -// Test the CLI argument parsing for external packages -describe("External Packages CLI Arguments", () => { - describe("--external-packages flag", () => { - it("should parse single package", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args(["--external-packages", "sqlite3"]); - expect(args.external_packages).toEqual(["sqlite3"]); - }); - - it("should parse multiple packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "sqlite3", - "fsevents", - "@mapbox/node-pre-gyp", - ]); - expect(args.external_packages).toEqual([ - "sqlite3", - "fsevents", - "@mapbox/node-pre-gyp", - ]); - }); - - it("should handle scoped packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "@scope/package", - "@another/scoped-package", - ]); - expect(args.external_packages).toEqual([ - "@scope/package", - "@another/scoped-package", - ]); - }); - - it("should handle packages with special characters", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "package-with-dashes", - "package.with.dots", - "@scope/package-with-dashes", - ]); - expect(args.external_packages).toEqual([ - "package-with-dashes", - "package.with.dots", - "@scope/package-with-dashes", - ]); - }); - - it("should handle empty flag (no packages)", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args(["--external-packages"]); - expect(args.external_packages).toEqual([]); - }); - - it("should handle flag not being used", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([]); - expect(args.external_packages).toBeUndefined(); - }); - - it("should handle mixed with other arguments", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - parser.add_argument("--verbose", { - action: "store_true", - help: "Verbose output", - }); - parser.add_argument("files", { - nargs: "*", - help: "Files to process", - }); - - const args = parser.parse_args([ - "--verbose", - "--external-packages", - "sqlite3", - "fsevents", - "--", - "file1.ts", - "file2.ts", - ]); - - expect(args.external_packages).toEqual(["sqlite3", "fsevents"]); - expect(args.verbose).toBe(true); - expect(args.files).toEqual(["file1.ts", "file2.ts"]); - }); - }); - - describe("Real-world scenarios", () => { - it("should handle database packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "sqlite3", - "better-sqlite3", - "mysql2", - "pg", - ]); - - expect(args.external_packages).toEqual([ - "sqlite3", - "better-sqlite3", - "mysql2", - "pg", - ]); - }); - - it("should handle native modules", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "sharp", - "canvas", - "node-sass", - "fsevents", - ]); - - expect(args.external_packages).toEqual([ - "sharp", - "canvas", - "node-sass", - "fsevents", - ]); - }); - - it("should handle problematic bundling packages", () => { - const parser = new ArgumentParser(); - parser.add_argument("--external-packages", { - nargs: "*", - help: "Additional packages to mark as external during bundling.", - }); - - const args = parser.parse_args([ - "--external-packages", - "@mapbox/node-pre-gyp", - "mock-aws-s3", - "aws-sdk", - "node-pre-gyp", - ]); - - expect(args.external_packages).toEqual([ - "@mapbox/node-pre-gyp", - "mock-aws-s3", - "aws-sdk", - "node-pre-gyp", - ]); - }); - }); -}); diff --git a/js/src/cli/util/pull.ts b/js/src/cli/util/pull.ts deleted file mode 100644 index 76d214237..000000000 --- a/js/src/cli/util/pull.ts +++ /dev/null @@ -1,394 +0,0 @@ -import { - Function as functionSchema, - type FunctionType as FunctionObject, - type SavedFunctionIdType as SavedFunctionId, - type ToolFunctionDefinitionType as ToolFunctionDefinition, -} from "../../generated_types"; -import { _internalGetGlobalState } from "../../logger"; -import { loadCLIEnv } from "./bundle"; -import { PullArgs } from "./types"; -import { warning } from "../../framework"; -import { z } from "zod/v3"; -import fs from "node:fs/promises"; -import util from "node:util"; -import { slugify } from "../../../util/string_util"; -import path from "node:path"; -import { runGitCommand } from "../../git-command"; -import { currentRepoPath } from "../../gitutil"; -import { isEmpty, loadPrettyXact, prettifyXact } from "../../../util/index"; -import { - ProjectNameIdMap, - toolFunctionDefinitionSchema, -} from "../../framework2"; -import pluralize from "pluralize"; - -export async function pullCommand(args: PullArgs) { - await loadCLIEnv(args); - - const loggerConn = _internalGetGlobalState().apiConn(); - const functions = await loggerConn.get_json("/v1/function", { - ...(args.project_id ? { project_id: args.project_id } : {}), - ...(args.project_name ? { project_name: args.project_name } : {}), - ...(args.slug ? { slug: args.slug } : {}), - ...(args.id ? { ids: [args.id] } : {}), - ...(args.version ? { version: loadPrettyXact(args.version) } : {}), - }); - const functionObjects = z - .object({ objects: z.array(z.unknown()) }) - .parse(functions); - - const projectNameToFunctions: Record = {}; - const projectNameIdMap = new ProjectNameIdMap(); - - for (const rawFunc of functionObjects.objects) { - const parsedFunc = functionSchema.safeParse(rawFunc); - if (!parsedFunc.success) { - const id = - typeof rawFunc === "object" && rawFunc && "id" in rawFunc - ? ` ${rawFunc.id}` - : ""; - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning(`Failed to parse function${id}: ${parsedFunc.error.message}`), - ); - continue; - } - - const func = parsedFunc.data; - const projectName = await projectNameIdMap.getName(func.project_id); - if (!projectNameToFunctions[projectName]) { - projectNameToFunctions[projectName] = []; - } - projectNameToFunctions[projectName].push(func); - } - - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log("Found functions in the following projects:"); - for (const projectName of Object.keys(projectNameToFunctions)) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log(` * ${projectName}`); - } - - const outputDir = args.output_dir ?? "./braintrust"; - await fs.mkdir(outputDir, { recursive: true }); - - const repoRoot = await currentRepoPath(); - const dirtyFileOutput = repoRoot - ? await runGitCommand(["diff", "--name-only", "-z", "HEAD"], { - cwd: repoRoot, - }) - : ""; - const dirtyFiles = new Set( - dirtyFileOutput - .split("\0") - .filter(Boolean) - .map((file) => path.resolve(repoRoot ?? ".", file)), - ); - - for (const projectName of Object.keys(projectNameToFunctions)) { - const projectFile = path.join( - outputDir, - `${slugify(projectName, { lower: true, strict: true, trim: true })}.ts`, - ); - const resolvedProjectFile = path.resolve(projectFile); - const fileExists = await fs.stat(projectFile).then( - () => true, - () => false, - ); - if (args.force) { - if (fileExists) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Overwriting ${doubleQuote(projectFile)} because --force is set.`, - ), - ); - } - } else if (dirtyFiles.has(resolvedProjectFile)) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Skipping project ${projectName} because ${doubleQuote(projectFile)} has uncommitted changes.`, - ), - ); - continue; - } else if (fileExists) { - if (!repoRoot) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Project ${projectName} already exists in ${doubleQuote(projectFile)}. Skipping since this is not a git repository...`, - ), - ); - continue; - } else { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Project ${projectName} already exists in ${doubleQuote(projectFile)}. Overwriting...`, - ), - ); - } - } - - const projectFileContents = await makeProjectFile({ - projectName, - projectId: await projectNameIdMap.getId(projectName), - fileName: projectFile, - functions: projectNameToFunctions[projectName], - hasSpecifiedFunction: !!args.slug || !!args.id, - }); - await fs.writeFile(projectFile, projectFileContents || ""); - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.log(`Wrote ${projectName} to ${doubleQuote(projectFile)}`); - } -} - -async function makeProjectFile({ - projectName, - projectId, - fileName, - functions, - hasSpecifiedFunction, -}: { - projectName: string; - projectId: string; - fileName: string; - functions: FunctionObject[]; - hasSpecifiedFunction: boolean; -}) { - const varNames = {}; - const functionDefinitions = functions - .map((f) => - makeFunctionDefinition({ func: f, varNames, hasSpecifiedFunction }), - ) - .filter((f) => f !== null); - const fileDef = `// This file was automatically generated by braintrust pull. You can -// generate it again by running: -// $ braintrust pull --project-name ${doubleQuote(projectName)} -// Feel free to edit this file manually, but once you do, you should make sure to -// sync your changes with Braintrust by running: -// $ braintrust push ${doubleQuote(fileName)} - -import braintrust from "braintrust"; - -const project = braintrust.projects.create({ - id: ${doubleQuote(projectId)}, - name: ${doubleQuote(projectName)}, -}); - -${functionDefinitions.join("\n")} -`; - - const prettier = await getPrettierModule(); - if (prettier) { - try { - const formatted = prettier.format(fileDef, { - parser: "typescript", - }); - return formatted; - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Failed to format with prettier (${error instanceof Error ? error.message : error}). Using unformatted output.`, - ), - ); - } - } - return fileDef; -} - -function makeFunctionDefinition({ - func, - varNames, - hasSpecifiedFunction, -}: { - func: FunctionObject; - varNames: Record; - hasSpecifiedFunction: boolean; -}): string | null { - if (func.function_data.type !== "prompt") { - if (hasSpecifiedFunction) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Skipping function ${doubleQuote(func.name)} because it is not a prompt.`, - ), - ); - } - return null; - } - - const baseVarName = slugToVarName(func.slug); - let varName = baseVarName; - let suffix = 1; - while (varName in varNames) { - varName = `${varName}${suffix}`; - suffix++; - } - varNames[varName] = func.slug; - - if (!func.prompt_data || !func.prompt_data.prompt) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Prompt ${doubleQuote(func.name)} has an invalid (empty) prompt definition.`, - ), - ); - return null; - } - const objectType = "prompt"; - const prompt = func.prompt_data.prompt; - const promptContents = - prompt.type === "completion" - ? `prompt: ${doubleQuote(prompt.content)}` - : `messages: ${safeStringify(prompt.messages).trimStart()}`; - - const rawToolsParsed = - prompt.type === "chat" && prompt.tools && prompt.tools.length > 0 - ? z - .array(toolFunctionDefinitionSchema) - .safeParse(JSON.parse(prompt.tools)) - : undefined; - - if (rawToolsParsed && !rawToolsParsed.success) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - `Prompt ${doubleQuote(func.name)} has an invalid tools definition: ${rawToolsParsed.error.message}. Skipping...`, - ), - ); - return null; - } - - const rawTools = rawToolsParsed ? rawToolsParsed.data : []; - - const { model, params } = func.prompt_data.options ?? {}; - - const paramsString = - params && Object.keys(params).length > 0 - ? `params: ${safeStringify(params).trimStart()},` - : ""; - - const tools: (SavedFunctionId | ToolFunctionDefinition)[] = [ - ...(func.prompt_data.tool_functions ?? []), - ...rawTools, - ]; - - const toolsString = - tools.length > 0 ? `tools: ${safeStringify(tools).trimStart()},` : ""; - - return `export const ${varName} = project.${pluralize(objectType)}.create({ - id: ${doubleQuote(func.id)}, - name: ${doubleQuote(func.name)}, - slug: ${doubleQuote(func.slug)}, - version: ${doubleQuote(prettifyXact(func._xact_id))}, ${printOptionalField("description", func.description)}${printOptionalField("model", model)} -${indent(promptContents, 2)}, -${indent(paramsString, 2)} -${indent(toolsString, 2)} -}); -`; -} - -function doubleQuote(str: string) { - return JSON.stringify(str); -} - -function slugToVarName(slug: string) { - let varName = slug.replace(/^[^a-zA-Z_$]|[^a-zA-Z0-9_$]/g, "_"); - varName = varName.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); - varName = varName.charAt(0).toLowerCase() + varName.slice(1); - return varName; -} - -function indent(str: string, numSpaces: number) { - return str.replace(/^/gm, " ".repeat(numSpaces)); -} - -function printOptionalField( - fieldName: string, - fieldValue: string | undefined | null, -) { - return !isEmpty(fieldValue) - ? ` - ${fieldName}: ${doubleQuote(fieldValue)},` - : ""; -} - -let prettierImportAttempted = false; - -let prettierModule: typeof import("prettier") | undefined = undefined; - -async function getPrettierModule() { - if (!prettierModule && !prettierImportAttempted) { - prettierImportAttempted = true; - - try { - // First try require() which is more stable in npx environments - prettierModule = require("prettier"); - } catch { - try { - // Fallback to dynamic import with error boundary - const importWithTimeout = () => { - return new Promise((resolve, reject) => { - let resolved = false; - - // Set a timeout to prevent infinite hanging - const timeoutId = setTimeout(() => { - if (!resolved) { - resolved = true; - reject(new Error("Prettier import timeout")); - } - }, 3000); - - import("prettier") - .then((module) => { - if (!resolved) { - resolved = true; - clearTimeout(timeoutId); - resolve(module); - } - }) - .catch((error) => { - if (!resolved) { - resolved = true; - clearTimeout(timeoutId); - reject(error); - } - }); - }); - }; - - prettierModule = await importWithTimeout(); - } catch { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn( - warning( - "Failed to load prettier module. Will not use prettier to format output.", - ), - ); - prettierModule = undefined; - } - } - } - return prettierModule; -} - -function safeStringify(obj: unknown): string { - try { - return JSON.stringify(obj, null, 2); - } catch (error) { - // Fallback for circular references or other JSON.stringify issues - try { - return util.inspect(obj, { - depth: 5, - maxStringLength: 1000, - breakLength: 80, - compact: false, - }); - } catch { - return `[Object: Unable to serialize - ${error instanceof Error ? error.message : error}]`; - } - } -} diff --git a/js/src/cli/util/types.ts b/js/src/cli/util/types.ts deleted file mode 100644 index 2909a607e..000000000 --- a/js/src/cli/util/types.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { type IfExistsType as IfExists } from "../../generated_types"; - -export interface CommonArgs { - /** @deprecated Use debug_logging: "debug" for detailed troubleshooting output. */ - verbose: boolean; - debug_logging?: "error" | "warn" | "info" | "debug"; -} - -export interface AuthArgs { - api_key?: string; - org_name?: string; - app_url?: string; - env_file?: string; -} - -export interface CompileArgs { - tsconfig?: string; - terminate_on_failure: boolean; - external_packages?: string[]; -} - -export interface RunArgs extends CommonArgs, AuthArgs, CompileArgs { - files: string[]; - watch: boolean; - list: boolean; - jsonl: boolean; - filter?: string[]; - no_send_logs: boolean; - no_progress_bars: boolean; - bundle: boolean; - push: boolean; - dev: boolean; - dev_host: string; - dev_port: number; - dev_org_name?: string; -} - -export interface BundleArgs extends CommonArgs, AuthArgs, CompileArgs { - files: string[]; - if_exists: IfExists; -} - -export interface PullArgs extends CommonArgs, AuthArgs { - output_dir: string; - project_name?: string; - project_id?: string; - id?: string; - slug?: string; - version?: string; - force: boolean; -} diff --git a/js/src/durable-eval.ts b/js/src/durable-eval.ts index 7a0c45e70..c27f63d26 100644 --- a/js/src/durable-eval.ts +++ b/js/src/durable-eval.ts @@ -852,7 +852,6 @@ async function startDurableEval< stop: () => undefined, increment: () => undefined, }, - [], undefined, parameters, true, diff --git a/js/src/eval-parameters.ts b/js/src/eval-parameters.ts index 1154fcfb8..bc15dc105 100644 --- a/js/src/eval-parameters.ts +++ b/js/src/eval-parameters.ts @@ -2,30 +2,25 @@ import { z } from "zod/v3"; import Ajv from "ajv"; import { Prompt, RemoteEvalParameters } from "./logger"; import { - promptDefinitionWithToolsSchema, promptDefinitionToPromptData, + type PromptDefinitionWithTools, } from "./prompt-schemas"; import { PromptData as promptDataSchema } from "./generated_types"; -// Schema for evaluation parameters -export const evalParametersSchema = z.record( - z.string(), - z.union([ - z.object({ - type: z.literal("prompt"), - default: promptDefinitionWithToolsSchema.optional(), - description: z.string().optional(), - }), - z.object({ - type: z.literal("model"), - default: z.string().optional(), - description: z.string().optional(), - }), - z.instanceof(z.ZodType), // For Zod schemas - ]), -); - -export type EvalParameters = z.infer; +export type EvalParameters = Record< + string, + | { + type: "prompt"; + default?: PromptDefinitionWithTools; + description?: string; + } + | { + type: "model"; + default?: string; + description?: string; + } + | z.ZodTypeAny +>; // Type helper to infer the type of a parameter value type InferParameterValue = T extends { type: "prompt" } diff --git a/js/src/exports.ts b/js/src/exports.ts index 2f4332842..487ec7430 100644 --- a/js/src/exports.ts +++ b/js/src/exports.ts @@ -1,3 +1,7 @@ +import type { z } from "zod/v3"; +import { AttachmentReference as attachmentReferenceSchema } from "./generated_types"; +import type { AttachmentReferenceType } from "./generated_plain_types"; + export type { AnyDataset, AttachmentParams, @@ -196,10 +200,7 @@ export { BraintrustMiddleware, wrapAISDKModel, } from "./wrappers/ai-sdk"; -export { - braintrustEveHook, - braintrustEveInstrumentation, -} from "./instrumentation/plugins/eve-plugin"; +export { braintrustEveInstrumentation } from "./instrumentation"; export { collectAnthropicSession } from "./wrappers/anthropic-session-collector"; export { wrapAnthropic } from "./wrappers/anthropic"; export { @@ -253,7 +254,6 @@ export type { EvalScorer, EvalClassifier, EvaluatorDef, - EvaluatorFile, ReporterBody, SpanContext, } from "./framework"; @@ -263,9 +263,6 @@ export { Eval, EvalResultWithSummary, Reporter, - buildLocalSummary, - reportFailures, - runEvaluator, defaultErrorScoreHandler, } from "./framework"; @@ -286,7 +283,6 @@ export { DatasetPipeline } from "./dataset-pipeline"; export type { CodeOpts, CreateProjectOpts, - FunctionEvent, PromptOpts, ScorerOpts, } from "./framework2"; @@ -295,12 +291,10 @@ export { CodeFunction, CodePrompt, Project, - ProjectNameIdMap, PromptBuilder, ScorerBuilder, ToolBuilder, projects, - toolFunctionDefinitionSchema, } from "./framework2"; export { @@ -332,27 +326,12 @@ export type { export { addAzureBlobHeaders, LazyValue } from "./util"; -export { AttachmentReference } from "./generated_types"; - -export type { - EvaluatorManifest, - EvalParameterSerializedSchema, - EvaluatorDefinition, - EvaluatorDefinitions, - ParametersSource, -} from "../dev/types"; +export const AttachmentReference: z.ZodType = + attachmentReferenceSchema; export type { EvalParameters } from "./eval-parameters"; -export { - evaluatorDefinitionSchema, - evaluatorDefinitionsSchema, -} from "../dev/types"; - // Auto-instrumentation configuration export { configureInstrumentation } from "./instrumentation"; -export { - braintrustFlueObserver, - braintrustFlueInstrumentation, -} from "./instrumentation"; +export { braintrustFlueInstrumentation } from "./instrumentation"; export type { InstrumentationConfig } from "./instrumentation"; diff --git a/js/src/framework-types.ts b/js/src/framework-types.ts index b5183cc7d..0f9d45c26 100644 --- a/js/src/framework-types.ts +++ b/js/src/framework-types.ts @@ -1,4 +1,4 @@ -import { type IfExistsType as IfExists } from "./generated_types"; +import type { IfExistsType as IfExists } from "./generated_plain_types"; export type GenericFunction = | ((input: Input) => Output) diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 4c86a8235..cb905aad7 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -54,7 +54,6 @@ test("meta (write) is passed to task", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -92,7 +91,6 @@ test("metadata (read/write) is passed to task", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -132,7 +130,6 @@ test("expected (read/write) is passed to task", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -175,7 +172,6 @@ test("EvalCase id and tags are passed to scorers", async () => { ], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -259,7 +255,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -292,7 +287,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -331,7 +325,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -369,7 +362,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], (event) => streamEvents.push(event), ); @@ -408,7 +400,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -432,7 +423,6 @@ describe("runEvaluator", () => { ), }, new NoopProgressReporter(), - [], undefined, ); @@ -459,7 +449,6 @@ describe("runEvaluator", () => { errorScoreHandler: defaultErrorScoreHandler, }, new NoopProgressReporter(), - [], undefined, ); @@ -488,7 +477,6 @@ describe("runEvaluator", () => { errorScoreHandler: defaultErrorScoreHandler, }, new NoopProgressReporter(), - [], undefined, ); @@ -521,7 +509,6 @@ describe("runEvaluator", () => { errorScoreHandler: () => undefined, }, new NoopProgressReporter(), - [], undefined, ); @@ -546,7 +533,6 @@ describe("runEvaluator", () => { errorScoreHandler: () => ({ error_score: 1 }), }, new NoopProgressReporter(), - [], undefined, ); @@ -579,7 +565,6 @@ describe("runEvaluator", () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -624,7 +609,6 @@ describe("runEvaluator", () => { maxConcurrency: 1, }, new NoopProgressReporter(), - [], undefined, ), ).rejects.toThrow(new InternalAbortError("Evaluator timed out")); @@ -673,7 +657,6 @@ describe("runEvaluator", () => { maxConcurrency: 1, }, new NoopProgressReporter(), - [], undefined, ), ).rejects.toThrow(new InternalAbortError("Evaluator aborted")); @@ -708,7 +691,6 @@ describe("runEvaluator", () => { scores: [], }, new NoopProgressReporter(), - [], undefined, ); @@ -740,7 +722,6 @@ describe("runEvaluator", () => { signal: abortController.signal, }, new NoopProgressReporter(), - [], undefined, ); @@ -769,7 +750,6 @@ test("trialIndex is passed to task", async () => { trialCount: 3, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -811,7 +791,6 @@ test("trialIndex with multiple inputs", async () => { trialCount: 2, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -857,7 +836,6 @@ test("per-input trialCount overrides global trialCount", async () => { trialCount: 2, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -904,7 +882,6 @@ test("per-input trialCount works without global trialCount", async () => { scores: [], }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1071,7 +1048,6 @@ test("runEvaluator forwards baseExperimentId to summary", async () => { baseExperimentId: "base-exp-id", }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1114,7 +1090,6 @@ test("runEvaluator forwards persisted baseExperimentName id to summary", async ( baseExperimentName: "base-exp", }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1156,7 +1131,6 @@ test("tags can be appended and logged to root span", async () => { summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1205,7 +1179,6 @@ test.each([ summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1245,7 +1218,6 @@ test("tags are persisted with a failing scorer", async () => { summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1278,7 +1250,6 @@ test("tags remain empty when not set", async () => { summarizeScores: false, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -1313,7 +1284,6 @@ test("scorer spans have purpose='scorer' attribute", async () => { ], }, new NoopProgressReporter(), - [], undefined, undefined, true, diff --git a/js/src/framework.ts b/js/src/framework.ts index 42d4e09ff..540b95dc6 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -7,13 +7,13 @@ import { SpanTypeAttribute, spanObjectTypeV3ToTypedString, } from "../util/index"; -import { - type GitMetadataSettingsType as GitMetadataSettings, - ObjectReference as ObjectReferenceSchema, - type ObjectReferenceType as ObjectReference, - type RepoInfoType as RepoInfo, - type SSEProgressEventDataType as SSEProgressEventData, -} from "./generated_types"; +import { ObjectReference as ObjectReferenceSchema } from "./generated_types"; +import type { + GitMetadataSettingsType as GitMetadataSettings, + ObjectReferenceType as ObjectReference, + RepoInfoType as RepoInfo, + SSEProgressEventDataType as SSEProgressEventData, +} from "./generated_plain_types"; import { queue } from "async"; import iso from "./isomorph"; @@ -121,7 +121,7 @@ export type EvalTask< hooks: EvalHooks, ) => Output); -export type TaskProgressEvent = Omit< +type TaskProgressEvent = Omit< SSEProgressEventData, "id" | "origin" | "object_type" | "name" >; @@ -404,7 +404,7 @@ export class EvalResultWithSummary< } } -export type { ReporterBody, ReporterDef } from "./reporters/types"; +export type { ReporterBody } from "./reporters/types"; async function getPersistedBaseExperimentId( experiment: Experiment, @@ -435,7 +435,7 @@ export type EvaluatorDef< evalName: string; } & Evaluator; -export type EvaluatorFile = { +type EvaluatorFile = { functions: CodeFunction< unknown, unknown, @@ -800,7 +800,7 @@ export async function Eval< if (typeof options.reporter === "string") { throw new Error( - "Must specify a reporter object, not a name. Can only specify reporter names when running 'braintrust eval'", + "Must specify a reporter object, not a name. Can only specify reporter names when running 'bt eval'", ); } @@ -847,7 +847,6 @@ export async function Eval< null, evalDef, progressReporter, - [], options.stream, options.parameters, shouldCollectResults, @@ -860,7 +859,6 @@ export async function Eval< experiment, evalDef, progressReporter, - [], options.stream, options.parameters, shouldCollectResults, @@ -903,64 +901,7 @@ export function Reporter( return ret; } -export interface Filter { - path: string[]; - pattern: RegExp; -} - -function serializeJSONWithPlainString(v: unknown) { - if (typeof v === "string") { - return v; - } else { - return JSON.stringify(v); - } -} - -function deserializePlainStringAsJSON(s: string) { - try { - return { value: JSON.parse(s), error: undefined }; - } catch (e) { - return { value: s, error: e }; - } -} - -export function parseFilters(filters: string[]): Filter[] { - const result: Filter[] = []; - for (const f of filters) { - const equalsIdx = f.indexOf("="); - if (equalsIdx === -1) { - throw new Error(`Invalid filter ${f}`); - } - const [path, value] = [f.slice(0, equalsIdx), f.slice(equalsIdx + 1)]; - let deserializedValue = deserializePlainStringAsJSON(value).value; - if (typeof deserializedValue !== "string") { - deserializedValue = value; // Just fall back to the original input - } - result.push({ - path: path.split("."), - pattern: new RegExp(deserializedValue), - }); - } - return result; -} - -function evaluateFilter(object: unknown, filter: Filter) { - const { path, pattern } = filter; - const key = path.reduce( - (acc, p) => - typeof acc === "object" && acc !== null - ? // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - (acc as Record)[p] - : undefined, - object, - ); - if (key === undefined) { - return false; - } - return pattern.test(serializeJSONWithPlainString(key)); -} - -export function scorerName( +function scorerName( // eslint-disable-next-line @typescript-eslint/no-explicit-any scorer: EvalScorer, scorer_idx: number, @@ -1207,7 +1148,6 @@ export async function runEvaluator( // eslint-disable-next-line @typescript-eslint/no-explicit-any evaluator: EvaluatorDef, progressReporter: ProgressReporter, - filters: Filter[], stream: ((data: SSEProgressEventData) => void) | undefined, parameters?: InferParameters, collectResults = true, @@ -1223,7 +1163,6 @@ export async function runEvaluator( experiment, evaluator, progressReporter, - filters, stream, parameters, collectResults, @@ -1246,7 +1185,6 @@ async function runEvaluatorInternal( // eslint-disable-next-line @typescript-eslint/no-explicit-any evaluator: EvaluatorDef, progressReporter: ProgressReporter, - filters: Filter[], stream: ((data: SSEProgressEventData) => void) | undefined, parameters: InferParameters | undefined, collectResults: boolean, @@ -1616,9 +1554,6 @@ async function runEvaluatorInternal( if (cancelled) { break; } - if (!filters.every((f) => evaluateFilter(datum, f))) { - continue; - } const trialCount = datum.trialCount ?? evaluator.trialCount ?? 1; for (let trialIndex = 0; trialIndex < trialCount; trialIndex++) { if (cancelled) { @@ -1756,10 +1691,9 @@ async function runEvaluatorInternal( } } -export const error = (text: string) => `Error: ${text}`; -export const warning = (text: string) => `Warning: ${text}`; +const warning = (text: string) => `Warning: ${text}`; -export function logError(e: unknown, verbose: boolean) { +function logError(e: unknown, verbose: boolean) { if (!verbose) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.error(`${e}`); @@ -1829,12 +1763,7 @@ export function buildLocalSummary( }; } -export function reportFailures< - Input, - Output, - Expected, - Metadata extends BaseMetadata, ->( +function reportFailures( evaluator: EvaluatorDef, failingResults: EvalResult[], { verbose, jsonl }: ReporterOpts, diff --git a/js/src/framework2.ts b/js/src/framework2.ts index 47994f5f1..ab4ee52c5 100644 --- a/js/src/framework2.ts +++ b/js/src/framework2.ts @@ -3,25 +3,23 @@ import type { Trace } from "./trace"; import iso from "./isomorph"; import { slugify } from "../util/string_util"; import { z } from "zod/v3"; -import { - type FunctionTypeEnumType as FunctionType, - type IfExistsType as IfExists, - type SavedFunctionIdType as SavedFunctionId, - type PromptBlockDataType as PromptBlockData, - type PromptDataType as PromptData, - ToolFunctionDefinition as toolFunctionDefinitionSchema, - type ToolFunctionDefinitionType as ToolFunctionDefinition, - FunctionData as functionDataSchema, - Project as projectSchema, - type ExtendedSavedFunctionIdType as ExtendedSavedFunctionId, -} from "./generated_types"; +import { Project as projectSchema } from "./generated_types"; +import type { + FunctionTypeEnumType as FunctionType, + IfExistsType as IfExists, + SavedFunctionIdType as SavedFunctionId, + PromptBlockDataType as PromptBlockData, + PromptDataType as PromptData, + ToolFunctionDefinitionType as ToolFunctionDefinition, + ExtendedSavedFunctionIdType as ExtendedSavedFunctionId, + FunctionDataType, +} from "./generated_plain_types"; import { loadPrettyXact, TransactionId } from "../util/index"; import { _internalGetGlobalState, login, Prompt, PromptRowWithId, - RemoteEvalParameters, } from "./logger"; import type { BaseFnOpts, GenericFunction } from "./framework-types"; import type { EvalParameters } from "./eval-parameters"; @@ -30,14 +28,12 @@ import { type PromptDefinition, } from "./prompt-schemas"; import { zodToJsonSchema } from "./zod/utils"; -import type { - ParametersSchema, - StaticParametersSchema, - SerializedParametersContainer, -} from "../dev/types"; - -export { toolFunctionDefinitionSchema }; -// ToolFunctionDefinition exported as type-only from main index to avoid namespace issues +type ParametersSchema = { + type: "object"; + properties: Record>; + required?: string[]; + additionalProperties?: boolean; +}; // Safe access to __filename (only exists in Node.js CJS) const currentFilename = @@ -120,7 +116,7 @@ export class Project { async publish() { if (globalThis._lazy_load) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.warn("publish() is a no-op when running `braintrust push`."); + console.warn("publish() is a no-op when running `bt push`."); return; } await login(); @@ -129,7 +125,7 @@ export class Project { if (this._publishableCodeFunctions.length > 0) { // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. console.warn( - "Code functions cannot be published directly. Use `braintrust push` instead.", + "Code functions cannot be published directly. Use `bt push` instead.", ); } if (this._publishablePrompts.length > 0) { @@ -710,53 +706,6 @@ class ParametersBuilder { } } -export function serializeEvalParametersToStaticParametersSchema( - parameters: EvalParameters, -): StaticParametersSchema { - return Object.fromEntries( - Object.entries(parameters).map(([name, value]) => { - if ("type" in value && value.type === "prompt") { - return [ - name, - { - type: "prompt", - default: value.default - ? promptDefinitionToPromptData(value.default) - : undefined, - description: value.description, - }, - ]; - } else if ("type" in value && value.type === "model") { - return [ - name, - { - type: "model", - default: value.default, - description: value.description, - }, - ]; - } else { - // Since this schema is bundled, it won't pass an instanceof check. For - // some reason, aliasing it to `z.ZodSchema` leads to `error TS2589: - // Type instantiation is excessively deep and possibly infinite.` So - // just using `any` to turn off the typesystem. - // - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - const schemaObj = zodToJsonSchema(value as unknown as z.ZodType); - return [ - name, - { - type: "data", - schema: schemaObj, - default: schemaObj.default, - description: schemaObj.description, - }, - ]; - } - }), - ); -} - function serializeEvalParameterstoParametersSchema( parameters: EvalParameters, ): ParametersSchema { @@ -827,38 +776,13 @@ function getDefaultDataFromParametersSchema( ); } -export function serializeRemoteEvalParametersContainer( - parameters: EvalParameters | RemoteEvalParameters, -): SerializedParametersContainer { - if (RemoteEvalParameters.isParameters(parameters)) { - return { - type: "braintrust.parameters", - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - schema: parameters.schema as ParametersSchema, - source: { - parametersId: parameters.id, - slug: parameters.slug, - name: parameters.name, - projectId: parameters.projectId, - version: parameters.version, - }, - }; - } - - return { - type: "braintrust.staticParameters", - schema: serializeEvalParametersToStaticParametersSchema(parameters), - source: null, - }; -} - -export interface FunctionEvent { +interface FunctionEvent { project_id: string; slug: string; name: string; description: string; prompt_data?: PromptData; - function_data: z.infer; + function_data: FunctionDataType; function_type?: FunctionType; if_exists?: IfExists; tags?: string[]; @@ -866,7 +790,7 @@ export interface FunctionEvent { environments?: { slug: string }[]; } -export class ProjectNameIdMap { +class ProjectNameIdMap { private nameToId: Record = {}; private idToName: Record = {}; diff --git a/js/src/functions/invoke.ts b/js/src/functions/invoke.ts index f7bbb35f5..82fd40de4 100644 --- a/js/src/functions/invoke.ts +++ b/js/src/functions/invoke.ts @@ -1,10 +1,10 @@ -import { - FunctionId as functionIdSchema, - type InvokeFunctionType as InvokeFunctionRequest, - type ChatCompletionMessageParamType as Message, - type StreamingModeType as StreamingMode, - type FunctionTypeEnumType as FunctionType, -} from "../generated_types"; +import { FunctionId as functionIdSchema } from "../generated_types"; +import type { + InvokeFunctionType as InvokeFunctionRequest, + ChatCompletionMessageParamType as Message, + StreamingModeType as StreamingMode, + FunctionTypeEnumType as FunctionType, +} from "../generated_plain_types"; import { _internalGetGlobalState, BraintrustState, diff --git a/js/src/functions/stream.ts b/js/src/functions/stream.ts index 300895075..392e6b337 100644 --- a/js/src/functions/stream.ts +++ b/js/src/functions/stream.ts @@ -1,9 +1,13 @@ import { - type CallEventType as CallEventSchema, CallEvent as callEventSchema, SSEConsoleEventData as sseConsoleEventDataSchema, SSEProgressEventData as sseProgressEventDataSchema, } from "../generated_types"; +import type { + CallEventType as CallEvent, + SSEConsoleEventDataType, + SSEProgressEventDataType, +} from "../generated_plain_types"; import { createParser, EventSourceParser, @@ -12,46 +16,55 @@ import { } from "eventsource-parser"; import { z } from "zod/v3"; -export const braintrustStreamChunkSchema = z.union([ - z.object({ - type: z.literal("text_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("reasoning_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("json_delta"), - data: z.string(), - }), - z.object({ - type: z.literal("error"), - data: z.string(), - }), - z.object({ - type: z.literal("console"), - data: sseConsoleEventDataSchema, - }), - z.object({ - type: z.literal("progress"), - data: sseProgressEventDataSchema, - }), - z.object({ - type: z.literal("start"), - data: z.string(), - }), - z.object({ - type: z.literal("done"), - data: z.string(), - }), -]); - /** * A chunk of data from a Braintrust stream. Each chunk type matches * an SSE event type. */ -export type BraintrustStreamChunk = z.infer; +export type BraintrustStreamChunk = + | { type: "text_delta"; data: string } + | { type: "reasoning_delta"; data: string } + | { type: "json_delta"; data: string } + | { type: "error"; data: string } + | { type: "console"; data: SSEConsoleEventDataType } + | { type: "progress"; data: SSEProgressEventDataType } + | { type: "start"; data: string } + | { type: "done"; data: string }; + +export const braintrustStreamChunkSchema: z.ZodType = + z.union([ + z.object({ + type: z.literal("text_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("reasoning_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("json_delta"), + data: z.string(), + }), + z.object({ + type: z.literal("error"), + data: z.string(), + }), + z.object({ + type: z.literal("console"), + data: sseConsoleEventDataSchema, + }), + z.object({ + type: z.literal("progress"), + data: sseProgressEventDataSchema, + }), + z.object({ + type: z.literal("start"), + data: z.string(), + }), + z.object({ + type: z.literal("done"), + data: z.string(), + }), + ]); /** * A Braintrust stream. This is a wrapper around a ReadableStream of `BraintrustStreamChunk`, @@ -163,7 +176,7 @@ export class BraintrustStream { return this.memoizedFinalValue; } - static parseRawEvent(event: CallEventSchema): BraintrustStreamChunk { + static parseRawEvent(event: CallEvent): BraintrustStreamChunk { switch (event.event) { case "text_delta": return { @@ -212,7 +225,7 @@ export class BraintrustStream { } } - static serializeRawEvent(event: BraintrustStreamChunk): CallEventSchema { + static serializeRawEvent(event: BraintrustStreamChunk): CallEvent { switch (event.type) { case "text_delta": return { diff --git a/js/src/gitutil.ts b/js/src/gitutil.ts index 7b23999af..9a687ebb3 100644 --- a/js/src/gitutil.ts +++ b/js/src/gitutil.ts @@ -1,7 +1,7 @@ -import { - type GitMetadataSettingsType as GitMetadataSettings, - type RepoInfoType as RepoInfo, -} from "./generated_types"; +import type { + GitMetadataSettingsType as GitMetadataSettings, + RepoInfoType as RepoInfo, +} from "./generated_plain_types"; import { debugLogger } from "./debug-logger"; import { runGitCommand } from "./git-command"; @@ -171,7 +171,7 @@ export async function getRepoInfo(settings?: GitMetadataSettings) { return sanitized; } -export async function currentRepoPath(): Promise { +async function currentRepoPath(): Promise { return await attempt(async () => (await runGitCommand(["rev-parse", "--show-toplevel"])).trim(), ); diff --git a/js/src/graph-framework.ts b/js/src/graph-framework.ts index f60ff3fba..5d6766da6 100644 --- a/js/src/graph-framework.ts +++ b/js/src/graph-framework.ts @@ -1,11 +1,11 @@ import { newId, Prompt } from "./logger"; -import { - type FunctionIdType as FunctionId, - type GraphDataType as GraphData, - type GraphNodeType as GraphNode, - type GraphEdgeType as GraphEdge, - type PromptBlockDataType as PromptBlockData, -} from "./generated_types"; +import type { + FunctionIdType as FunctionId, + GraphDataType as GraphData, + GraphNodeType as GraphNode, + GraphEdgeType as GraphEdge, + PromptBlockDataType as PromptBlockData, +} from "./generated_plain_types"; export interface BuildContext { getFunctionId(functionObj: unknown): Promise; diff --git a/js/src/imports.test.ts b/js/src/imports.test.ts index 96a4851d5..e00c08223 100644 --- a/js/src/imports.test.ts +++ b/js/src/imports.test.ts @@ -2,88 +2,7 @@ import { describe, it, expect } from "vitest"; import * as fs from "fs"; import * as path from "path"; -describe("CLI import restrictions", () => { - it("should not import from cli directory in non-cli code", () => { - const srcDir = path.join(__dirname); - const violations: string[] = []; - - function walkDirectory(dir: string) { - const entries = fs.readdirSync(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - const relativePath = path.relative(srcDir, fullPath); - - // Skip CLI directory and test files - if (relativePath.startsWith("cli/") || relativePath === "cli") { - continue; - } - - if (entry.isDirectory() && entry.name === "node_modules") { - continue; - } - - if (entry.isDirectory()) { - walkDirectory(fullPath); - } else if ( - entry.isFile() && - (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx")) && - !entry.name.endsWith(".d.ts") && - !entry.name.endsWith(".d.tsx") - ) { - checkFileForCliImports(fullPath, relativePath); - } - } - } - - function checkFileForCliImports(filePath: string, relativePath: string) { - const content = fs.readFileSync(filePath, "utf-8"); - const lines = content.split("\n"); - - lines.forEach((line, index) => { - // Check for import statements that reference CLI - const importMatch = - line.match(/from\s+["']([^"']+)["']/) || - line.match(/import\s*\(\s*["']([^"']+)["']\s*\)/); - - if (importMatch) { - const importPath = importMatch[1]; - - // Check if the import path references the CLI directory - // Only match actual /cli/ directory paths, not package names containing "cli" - // (e.g., @temporalio/client should not match) - if ( - importPath.includes("/cli/") || - importPath === "./cli" || - importPath === "../cli" || - importPath.endsWith("/cli") || - importPath.match(/^\.\.\/.*\/cli$/) || - importPath.match(/^\.\.\/.*\/cli\//) - ) { - violations.push( - `${relativePath}:${index + 1} - Illegal import from CLI: "${importPath}"`, - ); - } - } - }); - } - - walkDirectory(srcDir); - - if (violations.length > 0) { - const message = [ - "Found illegal imports from CLI directory in SDK code:", - "", - ...violations, - "", - "SDK code (src/**) must not import from CLI code (src/cli/**).", - "CLI code can import from SDK code, but not vice versa.", - ].join("\n"); - - expect.fail(message); - } - }); - +describe("Import restrictions", () => { it("should not allow eslint-disable comments for no-restricted-imports", () => { const srcDir = path.join(__dirname); const violations: string[] = []; @@ -95,11 +14,6 @@ describe("CLI import restrictions", () => { const fullPath = path.join(dir, entry.name); const relativePath = path.relative(srcDir, fullPath); - // Skip CLI directory and test files - if (relativePath.startsWith("cli/") || relativePath === "cli") { - continue; - } - if (entry.isDirectory() && entry.name === "node_modules") { continue; } @@ -153,7 +67,7 @@ describe("CLI import restrictions", () => { ...violations, "", "Disabling the no-restricted-imports rule is not allowed.", - "This rule prevents SDK code from importing CLI code and cannot be bypassed.", + "This rule protects SDK module boundaries and cannot be bypassed.", "If you believe you have a legitimate need for this import, please discuss with the team.", ].join("\n"); @@ -172,11 +86,6 @@ describe("CLI import restrictions", () => { const fullPath = path.join(dir, entry.name); const relativePath = path.relative(srcDir, fullPath); - // Skip the cli directory - CLI code is allowed to use require() and dynamic imports - if (entry.isDirectory() && entry.name === "cli") { - continue; - } - // Skip node_modules directories (test fixture deps, not SDK source) if (entry.isDirectory() && entry.name === "node_modules") { continue; @@ -224,11 +133,13 @@ describe("CLI import restrictions", () => { // Check for dynamic import() statements // Match import(...) but not static import statements - // Exception: allow dynamic import in anthropic-plugin for APIPromise patching + // Exception: allow dynamic import in anthropic-instrumentation for APIPromise patching if ( /\bimport\s*\(/.test(line) && !/^import\s+/.test(line.trim()) && - !relativePath.includes("instrumentation/plugins/anthropic-plugin.ts") + !relativePath.includes( + "instrumentation/providers/anthropic-instrumentation.ts", + ) ) { violations.push( `${relativePath}:${index + 1} - Found dynamic import() statement: "${line.trim()}"`, diff --git a/js/src/instrumentation/README.md b/js/src/instrumentation/README.md index 26487ec26..1fc368594 100644 --- a/js/src/instrumentation/README.md +++ b/js/src/instrumentation/README.md @@ -1,7 +1,8 @@ -# Writing Braintrust Instrumentation Plugins +# Braintrust Instrumentation Consumers -Braintrust instrumentation plugins wrap provider calls through typed invocation -hooks or consume tracing-compatible events from the internal global registry. +Braintrust instrumentation consumers wrap provider calls through typed +invocation hooks or consume tracing-compatible events from the internal global +registry. Auto-instrumented provider code and manual wrappers use the same typed channels, so extraction, stream handling, and span behavior stay aligned. @@ -13,15 +14,15 @@ An instrumentation has four parts: transformation. 2. A typed channel defines its arguments, result, extra event fields, and stable `orchestrion::` identifier. -3. A plugin intercepts that channel, or subscribes to its legacy tracing - lifecycle, and maps the call into Braintrust spans. +3. An internal consumer intercepts that channel, or subscribes to its legacy + tracing lifecycle, and maps the call into Braintrust spans. 4. A manual wrapper invokes the same typed channel when transformation is not available. -The global hook transport is internal. New and migrated plugins should prefer -the typed channel's `intercept` API. Existing plugins can continue using -`traceAsyncChannel`, `traceStreamingChannel`, `traceSyncStreamChannel`, or -`BasePlugin` helpers during the gradual migration. +The global hook transport and its consumers are internal. New and migrated +consumers should prefer the typed channel's `intercept` API. Existing consumers +can continue using `traceAsyncChannel`, `traceStreamingChannel`, or +`traceSyncStreamChannel` during the gradual migration. ## Invocation Hooks @@ -100,27 +101,25 @@ construct it from the package and operation. Prefer the shared tracing helpers: ```ts -this.register( - traceAsyncChannel(providerChannels.create, { - name: "provider.messages.create", - type: "llm", - extractInput(args) { - return { - input: args[0].messages, - metadata: { model: args[0].model }, - }; - }, - extractOutput(result) { - return result.content; - }, - extractMetrics(result) { - return { - prompt_tokens: result.usage.input_tokens, - completion_tokens: result.usage.output_tokens, - }; - }, - }), -); +traceAsyncChannel(providerChannels.create, { + name: "provider.messages.create", + type: "llm", + extractInput(args) { + return { + input: args[0].messages, + metadata: { model: args[0].model }, + }; + }, + extractOutput(result) { + return result.content; + }, + extractMetrics(result) { + return { + prompt_tokens: result.usage.input_tokens, + completion_tokens: result.usage.output_tokens, + }; + }, +}); ``` The helpers: @@ -129,7 +128,7 @@ The helpers: - bind the current span store to `start` for async-context propagation - contain extraction failures and log them through `debugLogger` - patch streams without replacing their public semantics -- unsubscribe and unbind stores when a plugin is disabled +- install process-lifetime subscriptions and span-store bindings Use raw `IsoChannelHandlers` only when a provider requires lifecycle behavior that the shared helpers cannot express. @@ -146,7 +145,7 @@ return providerChannels.create.invoke(originalCreate, this, [params], { ``` Legacy wrappers can continue calling the tracing-compatible operators until -their plugin is migrated: +their consumer is migrated: ```ts return providerChannels.create.tracePromise(() => originalCreate(params), { @@ -155,7 +154,7 @@ return providerChannels.create.tracePromise(() => originalCreate(params), { ``` Do not create spans directly inside wrappers. Keeping span creation in the -plugin prevents auto and manual instrumentation from drifting. +internal consumer prevents auto and manual instrumentation from drifting. ## Promise and Stream Requirements @@ -177,14 +176,14 @@ termination, and async context. - Avoid prototype-sensitive merges and unnecessary mutation of provider data. - Capture only fields permitted by the instrumentation specification. - Pass `Error` objects directly to `span.log({ error })`. -- Use narrow vendored provider interfaces shared by wrappers and plugins. -- Keep enable, disable, subscription, and patching behavior idempotent. +- Use narrow vendored provider interfaces shared by wrappers and consumers. +- Keep enable, subscription, and patching behavior idempotent. ## Testing Test at the narrowest useful layers: -1. Plugin unit tests for extraction and span handling. +1. Consumer unit tests for extraction and span handling. 2. Global hook/runtime tests for lifecycle and context behavior. 3. Orchestrion transformation tests for generated wrappers. 4. Bundler and loader tests for real transformed execution. diff --git a/js/src/instrumentation/auto-instrumentation-suppression.ts b/js/src/instrumentation/auto-instrumentation-suppression.ts index 2b31044ef..6002e0d1f 100644 --- a/js/src/instrumentation/auto-instrumentation-suppression.ts +++ b/js/src/instrumentation/auto-instrumentation-suppression.ts @@ -45,10 +45,10 @@ export function runWithAutoInstrumentationSuppressed(callback: () => R): R { export function bindAutoInstrumentationSuppressionToStart( tracingChannel: Pick, "start">, -): (() => void) | undefined { +): void { const startChannel = tracingChannel.start; if (!startChannel) { - return undefined; + return; } const store = suppressionStore(); @@ -61,10 +61,6 @@ export function bindAutoInstrumentationSuppressionToStart( }, ], })); - - return () => { - startChannel.unbindStore(store); - }; } export function enterAutoInstrumentationAllowed(): () => void { diff --git a/js/src/instrumentation/braintrust-plugin.test.ts b/js/src/instrumentation/braintrust-plugin.test.ts deleted file mode 100644 index 3365ec279..000000000 --- a/js/src/instrumentation/braintrust-plugin.test.ts +++ /dev/null @@ -1,1791 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; -import { BraintrustPlugin } from "./braintrust-plugin"; -import { OpenAIPlugin } from "./plugins/openai-plugin"; -import { OpenAICodexPlugin } from "./plugins/openai-codex-plugin"; -import { AnthropicPlugin } from "./plugins/anthropic-plugin"; -import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; -import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; -import { CloudflareThinkPlugin } from "./plugins/cloudflare-think-plugin"; -import { OpenAIAgentsPlugin } from "./plugins/openai-agents-plugin"; -import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; -import { HuggingFacePlugin } from "./plugins/huggingface-plugin"; -import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformers-plugin"; -import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; -import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; -import { MistralPlugin } from "./plugins/mistral-plugin"; -import { OllamaPlugin } from "./plugins/ollama-plugin"; -import { CoherePlugin } from "./plugins/cohere-plugin"; -import { GroqPlugin } from "./plugins/groq-plugin"; -import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; -import { LangChainPlugin } from "./plugins/langchain-plugin"; -import { LangSmithPlugin } from "./plugins/langsmith-plugin"; -import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; -import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; -import { VoyageAIPlugin } from "./plugins/voyageai-plugin"; -import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; -import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; - -function createPluginClassMock() { - return vi.fn(function MockPlugin(this: { - enable: ReturnType; - disable: ReturnType; - }) { - this.enable = vi.fn(); - this.disable = vi.fn(); - }); -} - -// Preserve the re-exported utility functions (parseMetricsFromUsage, etc.) -// while mocking out the OpenAIPlugin class — those utilities are also -// imported and tested at the bottom of this file via braintrust-plugin's -// re-exports. -vi.mock("./plugins/openai-plugin", async () => { - const actual = await vi.importActual< - typeof import("./plugins/openai-plugin") - >("./plugins/openai-plugin"); - return { - ...actual, - OpenAIPlugin: createPluginClassMock(), - }; -}); - -vi.mock("./plugins/anthropic-plugin", () => ({ - AnthropicPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openai-codex-plugin", () => ({ - OpenAICodexPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/ai-sdk-plugin", () => ({ - AISDKPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/claude-agent-sdk-plugin", () => ({ - ClaudeAgentSDKPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cloudflare-think-plugin", () => ({ - CloudflareThinkPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openai-agents-plugin", () => ({ - OpenAIAgentsPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/google-genai-plugin", () => ({ - GoogleGenAIPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/huggingface-plugin", () => ({ - HuggingFacePlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/huggingface-transformers-plugin", () => ({ - HuggingFaceTransformersPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openrouter-plugin", () => ({ - OpenRouterPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/openrouter-agent-plugin", () => ({ - OpenRouterAgentPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/mistral-plugin", () => ({ - MistralPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/ollama-plugin", () => ({ - OllamaPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cohere-plugin", () => ({ - CoherePlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/groq-plugin", () => ({ - GroqPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/github-copilot-plugin", () => ({ - GitHubCopilotPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/langchain-plugin", () => ({ - LangChainPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/langsmith-plugin", () => ({ - LangSmithPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/pi-coding-agent-plugin", () => ({ - PiCodingAgentPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/strands-agent-sdk-plugin", () => ({ - StrandsAgentSDKPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/voyageai-plugin", () => ({ - VoyageAIPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cloudflare-ai-chat-plugin", () => ({ - CloudflareAIChatPlugin: createPluginClassMock(), -})); - -vi.mock("./plugins/cloudflare-agents-plugin", () => ({ - CloudflareAgentsPlugin: createPluginClassMock(), -})); - -describe("BraintrustPlugin", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - describe("sub-plugin creation (default behavior)", () => { - it("should create and enable OpenAI plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Anthropic plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(AnthropicPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenAI Codex plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAICodexPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable AI SDK plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(AISDKPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Claude Agent SDK plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - const mockInstance = - vi.mocked(ClaudeAgentSDKPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cloudflare Think plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CloudflareThinkPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CloudflareThinkPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenAI Agents plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Google GenAI plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable HuggingFace plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(HuggingFacePlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - expect(HuggingFaceTransformersPlugin).toHaveBeenCalledTimes(1); - const transformersMockInstance = vi.mocked(HuggingFaceTransformersPlugin) - .mock.results[0].value; - expect(transformersMockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenRouter plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenRouterPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable OpenRouter Agent plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenRouterAgentPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Mistral plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(MistralPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(MistralPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cohere plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CoherePlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CoherePlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Voyage AI plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(VoyageAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(VoyageAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Groq plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(GroqPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(GroqPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable GitHubCopilot plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(GitHubCopilotPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(GitHubCopilotPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable LangChain plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(LangChainPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Strands Agent SDK plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(StrandsAgentSDKPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cloudflare AI Chat plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CloudflareAIChatPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CloudflareAIChatPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create and enable Cloudflare Agents plugin by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(CloudflareAgentsPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(CloudflareAgentsPlugin).mock.results[0] - .value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should create all plugins when enabled with no config", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(GitHubCopilotPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create all plugins when enabled with empty config", () => { - const plugin = new BraintrustPlugin({}); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(GitHubCopilotPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create all plugins when enabled with empty integrations config", () => { - const plugin = new BraintrustPlugin({ integrations: {} }); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - }); - }); - - describe("config-based disabling", () => { - it("should not create OpenAI plugin when openai: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openai: false }, - }); - plugin.enable(); - - expect(OpenAIPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Anthropic plugin when anthropic: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { anthropic: false }, - }); - plugin.enable(); - - expect(AnthropicPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create OpenAI Codex plugin when openaiCodexSDK: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openaiCodexSDK: false }, - }); - plugin.enable(); - - expect(OpenAICodexPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create AI SDK plugin when aisdk: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { aisdk: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Claude Agent SDK plugin when claudeAgentSDK: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { claudeAgentSDK: false }, - }); - plugin.enable(); - - expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cloudflare Think plugin when cloudflareThink: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cloudflareThink: false }, - }); - plugin.enable(); - - expect(CloudflareThinkPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create OpenAI Agents plugin when openAIAgents: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openAIAgents: false }, - }); - plugin.enable(); - - expect(OpenAIAgentsPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create HuggingFace plugin when huggingface: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { huggingface: false }, - }); - plugin.enable(); - - expect(HuggingFacePlugin).not.toHaveBeenCalled(); - expect(HuggingFaceTransformersPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Google GenAI plugin when googleGenAI: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { googleGenAI: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create OpenRouter plugin when openrouter: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openrouter: false }, - }); - plugin.enable(); - - expect(OpenRouterPlugin).not.toHaveBeenCalled(); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Mistral plugin when mistral: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { mistral: false }, - }); - plugin.enable(); - - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Ollama plugin when ollama: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { ollama: false }, - }); - plugin.enable(); - - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cohere plugin when cohere: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cohere: false }, - }); - plugin.enable(); - - expect(CoherePlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Groq plugin when groq: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { groq: false }, - }); - plugin.enable(); - - expect(GroqPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create GitHubCopilot plugin when gitHubCopilot: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { gitHubCopilot: false }, - }); - plugin.enable(); - - expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create LangChain plugin when langchain: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { langchain: false }, - }); - plugin.enable(); - - expect(LangChainPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create LangSmith with LangChain deduplication by default", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - expect(LangSmithPlugin).toHaveBeenCalledWith({ - skipLangChainRuns: true, - }); - const mockInstance = vi.mocked(LangSmithPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should disable LangSmith or let it capture LangChain runs", () => { - const disabled = new BraintrustPlugin({ - integrations: { langsmith: false }, - }); - disabled.enable(); - expect(LangSmithPlugin).not.toHaveBeenCalled(); - - const withoutLangChain = new BraintrustPlugin({ - integrations: { langchain: false }, - }); - withoutLangChain.enable(); - expect(LangSmithPlugin).toHaveBeenCalledWith({ - skipLangChainRuns: false, - }); - }); - - it("should not create OpenRouter Agent plugin when openrouterAgent: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { openrouterAgent: false }, - }); - plugin.enable(); - - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); - }); - - it("should not create any plugins when all are disabled", () => { - const plugin = new BraintrustPlugin({ - integrations: { - openai: false, - openaiCodexSDK: false, - anthropic: false, - aisdk: false, - claudeAgentSDK: false, - cloudflareThink: false, - openAIAgents: false, - googleGenAI: false, - huggingface: false, - openrouter: false, - openrouterAgent: false, - mistral: false, - ollama: false, - cohere: false, - groq: false, - gitHubCopilot: false, - langchain: false, - langsmith: false, - piCodingAgent: false, - strandsAgentSDK: false, - cloudflareAIChat: false, - cloudflareAgents: false, - }, - }); - plugin.enable(); - - expect(OpenAIPlugin).not.toHaveBeenCalled(); - expect(OpenAICodexPlugin).not.toHaveBeenCalled(); - expect(AnthropicPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).not.toHaveBeenCalled(); - expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); - expect(CloudflareThinkPlugin).not.toHaveBeenCalled(); - expect(OpenAIAgentsPlugin).not.toHaveBeenCalled(); - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - expect(HuggingFacePlugin).not.toHaveBeenCalled(); - expect(OpenRouterPlugin).not.toHaveBeenCalled(); - expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(CoherePlugin).not.toHaveBeenCalled(); - expect(GroqPlugin).not.toHaveBeenCalled(); - expect(GitHubCopilotPlugin).not.toHaveBeenCalled(); - expect(LangChainPlugin).not.toHaveBeenCalled(); - expect(LangSmithPlugin).not.toHaveBeenCalled(); - expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); - expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); - expect(CloudflareAIChatPlugin).not.toHaveBeenCalled(); - expect(CloudflareAgentsPlugin).not.toHaveBeenCalled(); - }); - - it("should not create Pi Coding Agent plugin when piCodingAgent: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { piCodingAgent: false }, - }); - plugin.enable(); - - expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Strands Agent SDK plugin when strandsAgentSDK: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { strandsAgentSDK: false }, - }); - plugin.enable(); - - expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cloudflare AI Chat plugin when cloudflareAIChat: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cloudflareAIChat: false }, - }); - plugin.enable(); - - expect(CloudflareAIChatPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Cloudflare Agents plugin when cloudflareAgents: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { cloudflareAgents: false }, - }); - plugin.enable(); - - expect(CloudflareAgentsPlugin).not.toHaveBeenCalled(); - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - }); - - it("should allow selective enabling of plugins", () => { - const plugin = new BraintrustPlugin({ - integrations: { - openai: true, - anthropic: false, - aisdk: false, - claudeAgentSDK: true, - openAIAgents: true, - googleGenAI: false, - huggingface: true, - openrouter: true, - mistral: false, - }, - }); - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).not.toHaveBeenCalled(); - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - expect(MistralPlugin).not.toHaveBeenCalled(); - }); - }); - - describe("backward compatibility (legacy config keys)", () => { - it("should not create AI SDK plugin when vercel: false (legacy)", () => { - const plugin = new BraintrustPlugin({ - integrations: { vercel: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create Google GenAI plugin when google: false (legacy)", () => { - const plugin = new BraintrustPlugin({ - integrations: { google: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - // Other plugins should still be created - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - }); - - it("should not create LangChain plugin when langgraph: false (alias)", () => { - const plugin = new BraintrustPlugin({ - integrations: { langgraph: false }, - }); - plugin.enable(); - - expect(LangChainPlugin).not.toHaveBeenCalled(); - expect(LangSmithPlugin).toHaveBeenCalledWith({ - skipLangChainRuns: true, - }); - }); - - it("should not create AI SDK plugin when both aisdk and vercel are false", () => { - const plugin = new BraintrustPlugin({ - integrations: { aisdk: false, vercel: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - }); - - it("should not create Google GenAI plugin when both googleGenAI and google are false", () => { - const plugin = new BraintrustPlugin({ - integrations: { googleGenAI: false, google: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - }); - - it("should not create AI SDK plugin when aisdk is true but vercel is false", () => { - const plugin = new BraintrustPlugin({ - integrations: { aisdk: true, vercel: false }, - }); - plugin.enable(); - - expect(AISDKPlugin).not.toHaveBeenCalled(); - }); - - it("should not create Google GenAI plugin when googleGenAI is true but google is false", () => { - const plugin = new BraintrustPlugin({ - integrations: { googleGenAI: true, google: false }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - }); - - it("should create AI SDK plugin when vercel is true and aisdk is not set", () => { - const plugin = new BraintrustPlugin({ - integrations: { vercel: true }, - }); - plugin.enable(); - - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - }); - - it("should create Google GenAI plugin when google is true and googleGenAI is not set", () => { - const plugin = new BraintrustPlugin({ - integrations: { google: true }, - }); - plugin.enable(); - - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - }); - }); - - describe("enable/disable lifecycle", () => { - it("should enable all sub-plugins when enabled", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - const openAICodexMock = - vi.mocked(OpenAICodexPlugin).mock.results[0].value; - const anthropicMock = vi.mocked(AnthropicPlugin).mock.results[0].value; - const aiSDKMock = vi.mocked(AISDKPlugin).mock.results[0].value; - const claudeAgentSDKMock = - vi.mocked(ClaudeAgentSDKPlugin).mock.results[0].value; - const openAIAgentsMock = - vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - const googleGenAIMock = - vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - const huggingFaceMock = - vi.mocked(HuggingFacePlugin).mock.results[0].value; - const huggingFaceTransformersMock = vi.mocked( - HuggingFaceTransformersPlugin, - ).mock.results[0].value; - const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; - const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock - .results[0].value; - const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; - const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; - const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; - const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; - const piCodingAgentMock = - vi.mocked(PiCodingAgentPlugin).mock.results[0].value; - const strandsAgentSDKMock = vi.mocked(StrandsAgentSDKPlugin).mock - .results[0].value; - const cloudflareAgentsMock = vi.mocked(CloudflareAgentsPlugin).mock - .results[0].value; - const langChainMock = vi.mocked(LangChainPlugin).mock.results[0].value; - - expect(openaiMock.enable).toHaveBeenCalledTimes(1); - expect(openAICodexMock.enable).toHaveBeenCalledTimes(1); - expect(anthropicMock.enable).toHaveBeenCalledTimes(1); - expect(aiSDKMock.enable).toHaveBeenCalledTimes(1); - expect(claudeAgentSDKMock.enable).toHaveBeenCalledTimes(1); - expect(openAIAgentsMock.enable).toHaveBeenCalledTimes(1); - expect(googleGenAIMock.enable).toHaveBeenCalledTimes(1); - expect(huggingFaceMock.enable).toHaveBeenCalledTimes(1); - expect(huggingFaceTransformersMock.enable).toHaveBeenCalledTimes(1); - expect(openRouterMock.enable).toHaveBeenCalledTimes(1); - expect(openRouterAgentMock.enable).toHaveBeenCalledTimes(1); - expect(mistralMock.enable).toHaveBeenCalledTimes(1); - expect(ollamaMock.enable).toHaveBeenCalledTimes(1); - expect(cohereMock.enable).toHaveBeenCalledTimes(1); - expect(groqMock.enable).toHaveBeenCalledTimes(1); - expect(piCodingAgentMock.enable).toHaveBeenCalledTimes(1); - expect(strandsAgentSDKMock.enable).toHaveBeenCalledTimes(1); - expect(cloudflareAgentsMock.enable).toHaveBeenCalledTimes(1); - expect(langChainMock.enable).toHaveBeenCalledTimes(1); - }); - - it("should disable and nullify all sub-plugins when disabled", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - const openAICodexMock = - vi.mocked(OpenAICodexPlugin).mock.results[0].value; - const anthropicMock = vi.mocked(AnthropicPlugin).mock.results[0].value; - const aiSDKMock = vi.mocked(AISDKPlugin).mock.results[0].value; - const claudeAgentSDKMock = - vi.mocked(ClaudeAgentSDKPlugin).mock.results[0].value; - const openAIAgentsMock = - vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - const googleGenAIMock = - vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - const huggingFaceMock = - vi.mocked(HuggingFacePlugin).mock.results[0].value; - const huggingFaceTransformersMock = vi.mocked( - HuggingFaceTransformersPlugin, - ).mock.results[0].value; - const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; - const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock - .results[0].value; - const mistralMock = vi.mocked(MistralPlugin).mock.results[0].value; - const ollamaMock = vi.mocked(OllamaPlugin).mock.results[0].value; - const cohereMock = vi.mocked(CoherePlugin).mock.results[0].value; - const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; - const piCodingAgentMock = - vi.mocked(PiCodingAgentPlugin).mock.results[0].value; - const strandsAgentSDKMock = vi.mocked(StrandsAgentSDKPlugin).mock - .results[0].value; - const cloudflareAgentsMock = vi.mocked(CloudflareAgentsPlugin).mock - .results[0].value; - const langChainMock = vi.mocked(LangChainPlugin).mock.results[0].value; - - plugin.disable(); - - expect(openaiMock.disable).toHaveBeenCalledTimes(1); - expect(openAICodexMock.disable).toHaveBeenCalledTimes(1); - expect(anthropicMock.disable).toHaveBeenCalledTimes(1); - expect(aiSDKMock.disable).toHaveBeenCalledTimes(1); - expect(claudeAgentSDKMock.disable).toHaveBeenCalledTimes(1); - expect(openAIAgentsMock.disable).toHaveBeenCalledTimes(1); - expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); - expect(huggingFaceMock.disable).toHaveBeenCalledTimes(1); - expect(huggingFaceTransformersMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); - expect(mistralMock.disable).toHaveBeenCalledTimes(1); - expect(ollamaMock.disable).toHaveBeenCalledTimes(1); - expect(cohereMock.disable).toHaveBeenCalledTimes(1); - expect(groqMock.disable).toHaveBeenCalledTimes(1); - expect(piCodingAgentMock.disable).toHaveBeenCalledTimes(1); - expect(strandsAgentSDKMock.disable).toHaveBeenCalledTimes(1); - expect(cloudflareAgentsMock.disable).toHaveBeenCalledTimes(1); - expect(langChainMock.disable).toHaveBeenCalledTimes(1); - }); - - it("should be idempotent on multiple enable calls", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - plugin.enable(); - plugin.enable(); - - // Should only create and enable plugins once - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - const mockInstance = vi.mocked(OpenAIPlugin).mock.results[0].value; - expect(mockInstance.enable).toHaveBeenCalledTimes(1); - }); - - it("should be idempotent on multiple disable calls", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - - plugin.disable(); - plugin.disable(); - plugin.disable(); - - // Should only disable plugins once - expect(openaiMock.disable).toHaveBeenCalledTimes(1); - }); - - it("should not disable plugins if never enabled", () => { - const plugin = new BraintrustPlugin(); - plugin.disable(); - - // Should not create any plugins - expect(OpenAIPlugin).not.toHaveBeenCalled(); - expect(OpenAICodexPlugin).not.toHaveBeenCalled(); - expect(AnthropicPlugin).not.toHaveBeenCalled(); - expect(AISDKPlugin).not.toHaveBeenCalled(); - expect(ClaudeAgentSDKPlugin).not.toHaveBeenCalled(); - expect(OpenAIAgentsPlugin).not.toHaveBeenCalled(); - expect(GoogleGenAIPlugin).not.toHaveBeenCalled(); - expect(HuggingFacePlugin).not.toHaveBeenCalled(); - expect(OpenRouterPlugin).not.toHaveBeenCalled(); - expect(OpenRouterAgentPlugin).not.toHaveBeenCalled(); - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(CoherePlugin).not.toHaveBeenCalled(); - expect(GroqPlugin).not.toHaveBeenCalled(); - expect(PiCodingAgentPlugin).not.toHaveBeenCalled(); - expect(StrandsAgentSDKPlugin).not.toHaveBeenCalled(); - }); - - it("should allow re-enabling after disable", () => { - const plugin = new BraintrustPlugin(); - plugin.enable(); - plugin.disable(); - - vi.clearAllMocks(); - - plugin.enable(); - - expect(OpenAIPlugin).toHaveBeenCalledTimes(1); - expect(OpenAICodexPlugin).toHaveBeenCalledTimes(1); - expect(AnthropicPlugin).toHaveBeenCalledTimes(1); - expect(AISDKPlugin).toHaveBeenCalledTimes(1); - expect(ClaudeAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(OpenAIAgentsPlugin).toHaveBeenCalledTimes(1); - expect(GoogleGenAIPlugin).toHaveBeenCalledTimes(1); - expect(HuggingFacePlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterPlugin).toHaveBeenCalledTimes(1); - expect(OpenRouterAgentPlugin).toHaveBeenCalledTimes(1); - expect(MistralPlugin).toHaveBeenCalledTimes(1); - expect(OllamaPlugin).toHaveBeenCalledTimes(1); - expect(CoherePlugin).toHaveBeenCalledTimes(1); - expect(GroqPlugin).toHaveBeenCalledTimes(1); - expect(PiCodingAgentPlugin).toHaveBeenCalledTimes(1); - expect(StrandsAgentSDKPlugin).toHaveBeenCalledTimes(1); - expect(LangChainPlugin).toHaveBeenCalledTimes(1); - expect(LangSmithPlugin).toHaveBeenCalledTimes(1); - }); - - it("should only disable plugins that were enabled", () => { - const plugin = new BraintrustPlugin({ - integrations: { - openai: true, - anthropic: false, - aisdk: true, - claudeAgentSDK: false, - openAIAgents: true, - googleGenAI: true, - huggingface: true, - openrouter: true, - openrouterAgent: true, - mistral: false, - ollama: false, - cohere: false, - groq: true, - langchain: true, - }, - }); - plugin.enable(); - - const openaiMock = vi.mocked(OpenAIPlugin).mock.results[0].value; - const aiSDKMock = vi.mocked(AISDKPlugin).mock.results[0].value; - const openAIAgentsMock = - vi.mocked(OpenAIAgentsPlugin).mock.results[0].value; - const googleGenAIMock = - vi.mocked(GoogleGenAIPlugin).mock.results[0].value; - const huggingFaceMock = - vi.mocked(HuggingFacePlugin).mock.results[0].value; - const openRouterMock = vi.mocked(OpenRouterPlugin).mock.results[0].value; - const openRouterAgentMock = vi.mocked(OpenRouterAgentPlugin).mock - .results[0].value; - const groqMock = vi.mocked(GroqPlugin).mock.results[0].value; - const langChainMock = vi.mocked(LangChainPlugin).mock.results[0].value; - - plugin.disable(); - - expect(openaiMock.disable).toHaveBeenCalledTimes(1); - expect(aiSDKMock.disable).toHaveBeenCalledTimes(1); - expect(openAIAgentsMock.disable).toHaveBeenCalledTimes(1); - expect(googleGenAIMock.disable).toHaveBeenCalledTimes(1); - expect(huggingFaceMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterMock.disable).toHaveBeenCalledTimes(1); - expect(openRouterAgentMock.disable).toHaveBeenCalledTimes(1); - expect(groqMock.disable).toHaveBeenCalledTimes(1); - expect(langChainMock.disable).toHaveBeenCalledTimes(1); - expect(MistralPlugin).not.toHaveBeenCalled(); - expect(OllamaPlugin).not.toHaveBeenCalled(); - expect(CoherePlugin).not.toHaveBeenCalled(); - }); - - it("should not create Voyage AI plugin when voyageai: false", () => { - const plugin = new BraintrustPlugin({ - integrations: { voyageai: false }, - }); - plugin.enable(); - - expect(VoyageAIPlugin).not.toHaveBeenCalled(); - }); - }); -}); - -// Re-exported utility function tests from OpenAI plugin -import { - parseMetricsFromUsage, - processImagesInOutput, - aggregateChatCompletionChunks, -} from "./braintrust-plugin"; -import { Attachment } from "../logger"; - -describe("parseMetricsFromUsage", () => { - it("should return empty object for null usage", () => { - expect(parseMetricsFromUsage(null)).toEqual({}); - }); - - it("should return empty object for undefined usage", () => { - expect(parseMetricsFromUsage(undefined)).toEqual({}); - }); - - it("should parse legacy token names", () => { - const usage = { - prompt_tokens: 10, - completion_tokens: 20, - total_tokens: 30, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 10, - completion_tokens: 20, - tokens: 30, // total_tokens is mapped to tokens - }); - }); - - it("should parse new API token names and map them", () => { - const usage = { - input_tokens: 15, - output_tokens: 25, - total_tokens: 40, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 15, // input_tokens mapped to prompt_tokens - completion_tokens: 25, // output_tokens mapped to completion_tokens - tokens: 40, // total_tokens mapped to tokens - }); - }); - - it("should parse token details fields", () => { - const usage = { - input_tokens: 100, - input_tokens_details: { - cached_tokens: 50, - audio_tokens: 10, - }, - output_tokens: 80, - output_tokens_details: { - reasoning_tokens: 20, - }, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 100, - prompt_cached_tokens: 50, // input -> prompt prefix - prompt_audio_tokens: 10, - completion_tokens: 80, - completion_reasoning_tokens: 20, // output -> completion prefix - }); - }); - - it("should handle mixed legacy and new token names", () => { - const usage = { - prompt_tokens: 10, - input_tokens: 15, - completion_tokens: 20, - total_tokens: 30, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 15, // Last one wins (input_tokens mapped) - completion_tokens: 20, - tokens: 30, // total_tokens is mapped to tokens - }); - }); - - it("should ignore non-number token values", () => { - const usage = { - prompt_tokens: "not a number", - completion_tokens: 20, - total_tokens: null, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - completion_tokens: 20, - }); - }); - - it("should ignore non-object token details", () => { - const usage = { - input_tokens: 100, - input_tokens_details: "not an object", - output_tokens: 80, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 100, - completion_tokens: 80, - }); - }); - - it("should ignore non-number values in token details", () => { - const usage = { - input_tokens: 100, - input_tokens_details: { - cached_tokens: 50, - invalid: "not a number", - also_invalid: null, - }, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - prompt_tokens: 100, - prompt_cached_tokens: 50, - }); - }); - - it("should handle empty usage object", () => { - expect(parseMetricsFromUsage({})).toEqual({}); - }); - - it("should handle unknown token prefix in details", () => { - const usage = { - custom_tokens: 100, - custom_tokens_details: { - special: 25, - }, - }; - - const result = parseMetricsFromUsage(usage); - - expect(result).toEqual({ - custom_tokens: 100, - custom_special: 25, // custom prefix preserved - }); - }); -}); - -describe("processImagesInOutput", () => { - it("should convert image_generation_call to attachment", () => { - // Create a small 1x1 red PNG base64 - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - output_format: "png", - revised_prompt: "A red pixel", - }; - - const result = processImagesInOutput(output); - - expect(result.type).toBe("image_generation_call"); - expect(result.output_format).toBe("png"); - expect(result.revised_prompt).toBe("A red pixel"); - expect(result.result).toBeInstanceOf(Attachment); - - const attachment = result.result as Attachment; - expect(attachment.reference.filename).toContain(".png"); - expect(attachment.reference.content_type).toBe("image/png"); - }); - - it("should handle image_generation_call with long revised_prompt", () => { - const longPrompt = - "This is a very long prompt that should be truncated to 50 characters when used as filename"; - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - revised_prompt: longPrompt, - }; - - const result = processImagesInOutput(output); - - const attachment = result.result as Attachment; - // Should be truncated to 50 chars + sanitized + extension - expect(attachment.reference.filename.length).toBeLessThanOrEqual(54); // 50 + ".png" - }); - - it("should handle image_generation_call without revised_prompt", () => { - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - output_format: "jpg", - }; - - const result = processImagesInOutput(output); - - const attachment = result.result as Attachment; - expect(attachment.reference.filename).toBe("generated_image.jpg"); - expect(attachment.reference.content_type).toBe("image/jpg"); - }); - - it("should use png as default format", () => { - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = { - type: "image_generation_call", - result: base64Image, - }; - - const result = processImagesInOutput(output); - - const attachment = result.result as Attachment; - expect(attachment.reference.filename).toBe("generated_image.png"); - expect(attachment.reference.content_type).toBe("image/png"); - }); - - it("should process arrays recursively", () => { - const base64Image = - "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg=="; - - const output = [ - { type: "text", content: "Hello" }, - { - type: "image_generation_call", - result: base64Image, - }, - { type: "text", content: "World" }, - ]; - - const result = processImagesInOutput(output); - - expect(Array.isArray(result)).toBe(true); - expect(result[0]).toEqual({ type: "text", content: "Hello" }); - expect(result[1].result).toBeInstanceOf(Attachment); - expect(result[2]).toEqual({ type: "text", content: "World" }); - }); - - it("should pass through non-image objects unchanged", () => { - const output = { - type: "text", - content: "Hello world", - }; - - const result = processImagesInOutput(output); - expect(result).toEqual(output); - }); - - it("should pass through primitive values unchanged", () => { - expect(processImagesInOutput("string")).toBe("string"); - expect(processImagesInOutput(42)).toBe(42); - expect(processImagesInOutput(true)).toBe(true); - expect(processImagesInOutput(null)).toBe(null); - }); - - it("should not process image_generation_call with non-string result", () => { - const output = { - type: "image_generation_call", - result: 12345, // Not a string - }; - - const result = processImagesInOutput(output); - expect(result).toEqual(output); - expect(result.result).toBe(12345); - }); -}); - -describe("aggregateChatCompletionChunks", () => { - it("should aggregate simple text chunks", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant", content: "Hello" } }], - }, - { - choices: [{ delta: { content: " world" } }], - }, - { - choices: [{ delta: { content: "!" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.output).toEqual([ - { - index: 0, - message: { - role: "assistant", - content: "Hello world!", - tool_calls: undefined, - }, - logprobs: null, - finish_reason: undefined, - }, - ]); - expect(result.metrics).toEqual({}); - }); - - it("should extract role from first chunk", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant" } }], - }, - { - choices: [{ delta: { content: "Hi" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.output[0].message.role).toBe("assistant"); - }); - - it("should extract finish_reason from last chunk with it", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant", content: "Done" } }], - }, - { - choices: [{ delta: { finish_reason: "stop" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.output[0].finish_reason).toBe("stop"); - }); - - it("should aggregate tool calls by id", () => { - const chunks = [ - { - choices: [ - { - delta: { - role: "assistant", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "get_weather", arguments: '{"loc' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: 'ation":"' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: 'NYC"}' }, - }, - ], - }, - }, - ], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.output[0].message.tool_calls).toEqual([ - { - id: "call_1", - type: "function", - function: { name: "get_weather", arguments: '{"location":"NYC"}' }, - }, - ]); - }); - - it("should handle multiple tool calls", () => { - const chunks = [ - { - choices: [ - { - delta: { - role: "assistant", - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "tool1", arguments: '{"a":' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: "1}" }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - id: "call_2", - type: "function", - function: { name: "tool2", arguments: '{"b":' }, - }, - ], - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - function: { arguments: "2}" }, - }, - ], - }, - }, - ], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - const output = result.output as any[]; - - expect(output[0].message.tool_calls).toHaveLength(2); - expect(output[0].message.tool_calls[0]).toEqual({ - id: "call_1", - type: "function", - function: { name: "tool1", arguments: '{"a":1}' }, - }); - expect(output[0].message.tool_calls[1]).toEqual({ - id: "call_2", - type: "function", - function: { name: "tool2", arguments: '{"b":2}' }, - }); - }); - - it("should parse usage metrics from chunks", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant", content: "Hi" } }], - }, - { - choices: [{ delta: { content: "!" } }], - usage: { - prompt_tokens: 10, - completion_tokens: 2, - total_tokens: 12, - }, - }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.metrics).toEqual({ - prompt_tokens: 10, - completion_tokens: 2, - tokens: 12, // total_tokens is mapped to tokens - }); - }); - - it("should merge usage from multiple chunks", () => { - const chunks = [ - { - choices: [{ delta: { role: "assistant" } }], - usage: { - prompt_tokens: 10, - }, - }, - { - choices: [{ delta: { content: "Hi" } }], - usage: { - completion_tokens: 5, - total_tokens: 15, - }, - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.metrics).toEqual({ - prompt_tokens: 10, - completion_tokens: 5, - tokens: 15, // total_tokens is mapped to tokens - }); - }); - - it("should handle empty chunks array", () => { - const result = aggregateChatCompletionChunks([]); - - expect(result.output).toEqual([ - { - index: 0, - message: { - role: undefined, - content: undefined, - tool_calls: undefined, - }, - logprobs: null, - finish_reason: undefined, - }, - ]); - expect(result.metrics).toEqual({}); - }); - - it("should handle chunks without choices", () => { - const chunks = [ - {}, - { choices: null }, - { choices: [] }, - { choices: [{ delta: { content: "Hi" } }] }, - ]; - - const result = aggregateChatCompletionChunks(chunks as any); - - expect(result.output[0].message.content).toBe("Hi"); - }); - - it("should handle chunks with only usage", () => { - const chunks = [ - { - usage: { - prompt_tokens: 10, - completion_tokens: 5, - }, - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.metrics).toEqual({ - prompt_tokens: 10, - completion_tokens: 5, - }); - }); - - it("should handle mixed content and tool calls", () => { - const chunks = [ - { - choices: [ - { - delta: { - role: "assistant", - content: "Let me check", - }, - }, - ], - }, - { - choices: [ - { - delta: { - tool_calls: [ - { - id: "call_1", - type: "function", - function: { name: "check", arguments: "{}" }, - }, - ], - }, - }, - ], - }, - { - choices: [{ delta: { finish_reason: "tool_calls" } }], - }, - ]; - - const result = aggregateChatCompletionChunks(chunks); - - expect(result.output[0].message.content).toBe("Let me check"); - expect(result.output[0].message.tool_calls).toHaveLength(1); - expect(result.output[0].finish_reason).toBe("tool_calls"); - }); -}); diff --git a/js/src/instrumentation/braintrust-plugin.ts b/js/src/instrumentation/braintrust-plugin.ts deleted file mode 100644 index 4ed98d53e..000000000 --- a/js/src/instrumentation/braintrust-plugin.ts +++ /dev/null @@ -1,414 +0,0 @@ -import { BasePlugin } from "./core"; -import { OpenAIPlugin } from "./plugins/openai-plugin"; -import { OpenAICodexPlugin } from "./plugins/openai-codex-plugin"; -import { AnthropicPlugin } from "./plugins/anthropic-plugin"; -import { AISDKPlugin } from "./plugins/ai-sdk-plugin"; -import { ClaudeAgentSDKPlugin } from "./plugins/claude-agent-sdk-plugin"; -import { CloudflareThinkPlugin } from "./plugins/cloudflare-think-plugin"; -import { CursorSDKPlugin } from "./plugins/cursor-sdk-plugin"; -import { OpenAIAgentsPlugin } from "./plugins/openai-agents-plugin"; -import { GoogleGenAIPlugin } from "./plugins/google-genai-plugin"; -import { HuggingFacePlugin } from "./plugins/huggingface-plugin"; -import { HuggingFaceTransformersPlugin } from "./plugins/huggingface-transformers-plugin"; -import { OpenRouterAgentPlugin } from "./plugins/openrouter-agent-plugin"; -import { OpenRouterPlugin } from "./plugins/openrouter-plugin"; -import { MistralPlugin } from "./plugins/mistral-plugin"; -import { OllamaPlugin } from "./plugins/ollama-plugin"; -import { GoogleADKPlugin } from "./plugins/google-adk-plugin"; -import { CoherePlugin } from "./plugins/cohere-plugin"; -import { GroqPlugin } from "./plugins/groq-plugin"; -import { BedrockRuntimePlugin } from "./plugins/bedrock-runtime-plugin"; -import { GenkitPlugin } from "./plugins/genkit-plugin"; -import { GitHubCopilotPlugin } from "./plugins/github-copilot-plugin"; -import { FluePlugin } from "./plugins/flue-plugin"; -import { LangChainPlugin } from "./plugins/langchain-plugin"; -import { LangSmithPlugin } from "./plugins/langsmith-plugin"; -import { PiCodingAgentPlugin } from "./plugins/pi-coding-agent-plugin"; -import { StrandsAgentSDKPlugin } from "./plugins/strands-agent-sdk-plugin"; -import { VoyageAIPlugin } from "./plugins/voyageai-plugin"; -import { CloudflareAIChatPlugin } from "./plugins/cloudflare-ai-chat-plugin"; -import { CloudflareAgentsPlugin } from "./plugins/cloudflare-agents-plugin"; -import type { InstrumentationIntegrationsConfig } from "./config"; - -export interface BraintrustPluginConfig { - integrations?: InstrumentationIntegrationsConfig; -} - -/** - * Default Braintrust plugin that manages all AI provider instrumentation plugins. - * - * This plugin orchestrates: - * - OpenAI SDK (chat completions, embeddings, etc.) - * - Anthropic SDK (messages) - * - Claude Agent SDK (agent interactions) - * - Vercel AI SDK (generateText, streamText, etc.) - * - Google GenAI SDK - * - HuggingFace Inference SDK - * - LangChain.js and LangGraph - * - Mistral SDK - * - Ollama SDK - * - Cohere SDK - * - Voyage AI SDK - * - * The plugin is automatically enabled when the Braintrust library is loaded. - * Individual integrations can be disabled via configuration. - */ -export class BraintrustPlugin extends BasePlugin { - private config: BraintrustPluginConfig; - private openaiPlugin: OpenAIPlugin | null = null; - private openAICodexPlugin: OpenAICodexPlugin | null = null; - private anthropicPlugin: AnthropicPlugin | null = null; - private aiSDKPlugin: AISDKPlugin | null = null; - private claudeAgentSDKPlugin: ClaudeAgentSDKPlugin | null = null; - private cloudflareThinkPlugin: CloudflareThinkPlugin | null = null; - private cursorSDKPlugin: CursorSDKPlugin | null = null; - private openAIAgentsPlugin: OpenAIAgentsPlugin | null = null; - private googleGenAIPlugin: GoogleGenAIPlugin | null = null; - private huggingFacePlugin: HuggingFacePlugin | null = null; - private huggingFaceTransformersPlugin: HuggingFaceTransformersPlugin | null = - null; - private openRouterPlugin: OpenRouterPlugin | null = null; - private openRouterAgentPlugin: OpenRouterAgentPlugin | null = null; - private mistralPlugin: MistralPlugin | null = null; - private ollamaPlugin: OllamaPlugin | null = null; - private googleADKPlugin: GoogleADKPlugin | null = null; - private coherePlugin: CoherePlugin | null = null; - private groqPlugin: GroqPlugin | null = null; - private bedrockRuntimePlugin: BedrockRuntimePlugin | null = null; - private genkitPlugin: GenkitPlugin | null = null; - private gitHubCopilotPlugin: GitHubCopilotPlugin | null = null; - private fluePlugin: FluePlugin | null = null; - private langChainPlugin: LangChainPlugin | null = null; - private langSmithPlugin: LangSmithPlugin | null = null; - private piCodingAgentPlugin: PiCodingAgentPlugin | null = null; - private strandsAgentSDKPlugin: StrandsAgentSDKPlugin | null = null; - private voyageAIPlugin: VoyageAIPlugin | null = null; - private cloudflareAIChatPlugin: CloudflareAIChatPlugin | null = null; - private cloudflareAgentsPlugin: CloudflareAgentsPlugin | null = null; - - constructor(config: BraintrustPluginConfig = {}) { - super(); - this.config = config; - } - - protected onEnable(): void { - const integrations = this.config.integrations ?? {}; - - // Enable OpenAI integration (default: true) - if (integrations.openai !== false) { - this.openaiPlugin = new OpenAIPlugin(); - this.openaiPlugin.enable(); - } - - if (integrations.openaiCodexSDK !== false) { - this.openAICodexPlugin = new OpenAICodexPlugin(); - this.openAICodexPlugin.enable(); - } - - // Enable Anthropic integration (default: true) - if (integrations.anthropic !== false) { - this.anthropicPlugin = new AnthropicPlugin(); - this.anthropicPlugin.enable(); - } - - // Enable AI SDK integration (default: true) - // Support both 'aisdk' and legacy 'vercel' config keys - if (integrations.aisdk !== false && integrations.vercel !== false) { - this.aiSDKPlugin = new AISDKPlugin(); - this.aiSDKPlugin.enable(); - } - - // Enable Claude Agent SDK integration (default: true) - if (integrations.claudeAgentSDK !== false) { - this.claudeAgentSDKPlugin = new ClaudeAgentSDKPlugin(); - this.claudeAgentSDKPlugin.enable(); - } - - if (integrations.cloudflareThink !== false) { - this.cloudflareThinkPlugin = new CloudflareThinkPlugin(); - this.cloudflareThinkPlugin.enable(); - } - - if (integrations.cursorSDK !== false && integrations.cursor !== false) { - this.cursorSDKPlugin = new CursorSDKPlugin(); - this.cursorSDKPlugin.enable(); - } - - // Enable OpenAI Agents SDK integration (default: true) - if (integrations.openAIAgents !== false) { - this.openAIAgentsPlugin = new OpenAIAgentsPlugin(); - this.openAIAgentsPlugin.enable(); - } - - // Enable Google GenAI integration (default: true) - // Support both 'googleGenAI' and legacy 'google' config keys - if (integrations.googleGenAI !== false && integrations.google !== false) { - this.googleGenAIPlugin = new GoogleGenAIPlugin(); - this.googleGenAIPlugin.enable(); - } - - if (integrations.huggingface !== false) { - this.huggingFacePlugin = new HuggingFacePlugin(); - this.huggingFacePlugin.enable(); - this.huggingFaceTransformersPlugin = new HuggingFaceTransformersPlugin(); - this.huggingFaceTransformersPlugin.enable(); - } - - if (integrations.openrouter !== false) { - this.openRouterPlugin = new OpenRouterPlugin(); - this.openRouterPlugin.enable(); - } - - if (integrations.openrouterAgent !== false) { - this.openRouterAgentPlugin = new OpenRouterAgentPlugin(); - this.openRouterAgentPlugin.enable(); - } - - if (integrations.mistral !== false) { - this.mistralPlugin = new MistralPlugin(); - this.mistralPlugin.enable(); - } - - if (integrations.ollama !== false) { - this.ollamaPlugin = new OllamaPlugin(); - this.ollamaPlugin.enable(); - } - - // Enable Google ADK integration (default: true) - if (integrations.googleADK !== false) { - this.googleADKPlugin = new GoogleADKPlugin(); - this.googleADKPlugin.enable(); - } - - if (integrations.cohere !== false) { - this.coherePlugin = new CoherePlugin(); - this.coherePlugin.enable(); - } - - if (integrations.voyageai !== false) { - this.voyageAIPlugin = new VoyageAIPlugin(); - this.voyageAIPlugin.enable(); - } - - if (integrations.groq !== false) { - this.groqPlugin = new GroqPlugin(); - this.groqPlugin.enable(); - } - - if ( - integrations.bedrock !== false && - integrations.awsBedrock !== false && - integrations.awsBedrockRuntime !== false - ) { - this.bedrockRuntimePlugin = new BedrockRuntimePlugin(); - this.bedrockRuntimePlugin.enable(); - } - - if (integrations.genkit !== false) { - this.genkitPlugin = new GenkitPlugin(); - this.genkitPlugin.enable(); - } - - if (integrations.gitHubCopilot !== false) { - this.gitHubCopilotPlugin = new GitHubCopilotPlugin(); - this.gitHubCopilotPlugin.enable(); - } - - if (integrations.piCodingAgent !== false) { - this.piCodingAgentPlugin = new PiCodingAgentPlugin(); - this.piCodingAgentPlugin.enable(); - } - - if (integrations.strandsAgentSDK !== false) { - this.strandsAgentSDKPlugin = new StrandsAgentSDKPlugin(); - this.strandsAgentSDKPlugin.enable(); - } - - if (integrations.cloudflareAIChat !== false) { - this.cloudflareAIChatPlugin = new CloudflareAIChatPlugin(); - this.cloudflareAIChatPlugin.enable(); - } - - if (integrations.cloudflareAgents !== false) { - this.cloudflareAgentsPlugin = new CloudflareAgentsPlugin(); - this.cloudflareAgentsPlugin.enable(); - } - - if (integrations.flue !== false) { - this.fluePlugin = new FluePlugin(); - this.fluePlugin.enable(); - } - - if (integrations.langchain !== false && integrations.langgraph !== false) { - this.langChainPlugin = new LangChainPlugin(); - this.langChainPlugin.enable(); - } - - if (integrations.langsmith !== false) { - this.langSmithPlugin = new LangSmithPlugin({ - skipLangChainRuns: integrations.langchain !== false, - }); - this.langSmithPlugin.enable(); - } - - // Mastra is intentionally not wired here: `@mastra/core` ships its own - // ObservabilityExporter contract, and `BraintrustObservabilityExporter` - // (wrappers/mastra.ts) is auto-installed by the loader patch in - // `auto-instrumentations/loader/mastra-observability-patch.ts` rather than - // by a BasePlugin / global hook subscription. - } - - protected onDisable(): void { - if (this.openaiPlugin) { - this.openaiPlugin.disable(); - this.openaiPlugin = null; - } - - if (this.openAICodexPlugin) { - this.openAICodexPlugin.disable(); - this.openAICodexPlugin = null; - } - - if (this.anthropicPlugin) { - this.anthropicPlugin.disable(); - this.anthropicPlugin = null; - } - - if (this.aiSDKPlugin) { - this.aiSDKPlugin.disable(); - this.aiSDKPlugin = null; - } - - if (this.claudeAgentSDKPlugin) { - this.claudeAgentSDKPlugin.disable(); - this.claudeAgentSDKPlugin = null; - } - - if (this.cloudflareThinkPlugin) { - this.cloudflareThinkPlugin.disable(); - this.cloudflareThinkPlugin = null; - } - - if (this.cursorSDKPlugin) { - this.cursorSDKPlugin.disable(); - this.cursorSDKPlugin = null; - } - - if (this.openAIAgentsPlugin) { - this.openAIAgentsPlugin.disable(); - this.openAIAgentsPlugin = null; - } - - if (this.googleGenAIPlugin) { - this.googleGenAIPlugin.disable(); - this.googleGenAIPlugin = null; - } - - if (this.huggingFacePlugin) { - this.huggingFacePlugin.disable(); - this.huggingFacePlugin = null; - } - - if (this.huggingFaceTransformersPlugin) { - this.huggingFaceTransformersPlugin.disable(); - this.huggingFaceTransformersPlugin = null; - } - - if (this.openRouterPlugin) { - this.openRouterPlugin.disable(); - this.openRouterPlugin = null; - } - - if (this.openRouterAgentPlugin) { - this.openRouterAgentPlugin.disable(); - this.openRouterAgentPlugin = null; - } - - if (this.mistralPlugin) { - this.mistralPlugin.disable(); - this.mistralPlugin = null; - } - - if (this.ollamaPlugin) { - this.ollamaPlugin.disable(); - this.ollamaPlugin = null; - } - - if (this.googleADKPlugin) { - this.googleADKPlugin.disable(); - this.googleADKPlugin = null; - } - - if (this.coherePlugin) { - this.coherePlugin.disable(); - this.coherePlugin = null; - } - - if (this.voyageAIPlugin) { - this.voyageAIPlugin.disable(); - this.voyageAIPlugin = null; - } - - if (this.groqPlugin) { - this.groqPlugin.disable(); - this.groqPlugin = null; - } - - if (this.bedrockRuntimePlugin) { - this.bedrockRuntimePlugin.disable(); - this.bedrockRuntimePlugin = null; - } - - if (this.genkitPlugin) { - this.genkitPlugin.disable(); - this.genkitPlugin = null; - } - - if (this.gitHubCopilotPlugin) { - this.gitHubCopilotPlugin.disable(); - this.gitHubCopilotPlugin = null; - } - - if (this.piCodingAgentPlugin) { - this.piCodingAgentPlugin.disable(); - this.piCodingAgentPlugin = null; - } - - if (this.strandsAgentSDKPlugin) { - this.strandsAgentSDKPlugin.disable(); - this.strandsAgentSDKPlugin = null; - } - - if (this.cloudflareAIChatPlugin) { - this.cloudflareAIChatPlugin.disable(); - this.cloudflareAIChatPlugin = null; - } - - if (this.cloudflareAgentsPlugin) { - this.cloudflareAgentsPlugin.disable(); - this.cloudflareAgentsPlugin = null; - } - - if (this.fluePlugin) { - this.fluePlugin.disable(); - this.fluePlugin = null; - } - - if (this.langChainPlugin) { - this.langChainPlugin.disable(); - this.langChainPlugin = null; - } - - if (this.langSmithPlugin) { - this.langSmithPlugin.disable(); - this.langSmithPlugin = null; - } - } -} - -// Re-export utility functions from OpenAIPlugin for backward compatibility -export { - parseMetricsFromUsage, - processImagesInOutput, - aggregateChatCompletionChunks, -} from "./plugins/openai-plugin"; diff --git a/js/src/instrumentation/config.ts b/js/src/instrumentation/config.ts index 35d3d2cb3..e8df49037 100644 --- a/js/src/instrumentation/config.ts +++ b/js/src/instrumentation/config.ts @@ -12,7 +12,6 @@ export interface InstrumentationIntegrationsConfig { cloudflareThink?: boolean; cursor?: boolean; cursorSDK?: boolean; - flue?: boolean; mastra?: boolean; openAIAgents?: boolean; openrouter?: boolean; @@ -80,8 +79,6 @@ const envIntegrationAliases: Record< cursor: "cursor", "cursor-sdk": "cursorSDK", cursorsdk: "cursorSDK", - flue: "flue", - "flue-runtime": "flue", mastra: "mastra", "openai-agents": "openAIAgents", openaiagents: "openAIAgents", @@ -143,7 +140,6 @@ export function getDefaultInstrumentationIntegrations(): Record< cloudflareThink: true, cursor: true, cursorSDK: true, - flue: true, mastra: true, openAIAgents: true, openrouter: true, diff --git a/js/src/instrumentation/core/channel-tracing.test.ts b/js/src/instrumentation/core/channel-tracing.test.ts index 55c08f05b..319d24344 100644 --- a/js/src/instrumentation/core/channel-tracing.test.ts +++ b/js/src/instrumentation/core/channel-tracing.test.ts @@ -27,12 +27,32 @@ import { traceAsyncChannel, traceStreamingChannel } from "./channel-tracing"; const testChannels = defineChannels( "channel-tracing-test", { - asyncCall: channel<[Record], { ok: true }>({ - channelName: "async.call", + asyncBinding: channel<[Record], { ok: true }>({ + channelName: "async.binding", kind: "async", }), - streamingCall: channel<[Record], { ok: true }>({ - channelName: "streaming.call", + provenance: channel<[Record], { ok: true }>({ + channelName: "async.provenance", + kind: "async", + }), + skipped: channel<[Record], { ok: true }>({ + channelName: "async.skipped", + kind: "async", + }), + throwingPredicate: channel<[Record], { ok: true }>({ + channelName: "async.throwing-predicate", + kind: "async", + }), + suppressed: channel<[Record], { ok: true }>({ + channelName: "async.suppressed", + kind: "async", + }), + streamingCleanup: channel<[Record], { ok: true }>({ + channelName: "streaming.cleanup", + kind: "async", + }), + streamingCancellation: channel<[Record], { ok: true }>({ + channelName: "streaming.cancellation", kind: "async", }), }, @@ -60,7 +80,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("binds the created span into the traced async execution context", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.asyncBinding, { name: "channel-tracing-test", type: "function", extractInput: () => ({ @@ -73,21 +93,16 @@ describe("traceAsyncChannel current span binding", () => { const seenSpanIds: string[] = []; - try { - await testChannels.asyncCall.tracePromise( - async () => { - seenSpanIds.push(currentSpan().spanId); - await Promise.resolve(); - seenSpanIds.push(currentSpan().spanId); - - return { ok: true as const }; - }, - { arguments: [{}] } as any, - ); - } finally { - unsubscribe(); - } + await testChannels.asyncBinding.tracePromise( + async () => { + seenSpanIds.push(currentSpan().spanId); + await Promise.resolve(); + seenSpanIds.push(currentSpan().spanId); + return { ok: true as const }; + }, + { arguments: [{}] } as any, + ); expect(seenSpanIds).toHaveLength(2); expect(seenSpanIds[0]).toBeTruthy(); expect(seenSpanIds[1]).toBe(seenSpanIds[0]); @@ -103,7 +118,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("limits channel provenance to directly instrumented spans", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.provenance, { name: "channel-parent", type: "function", extractInput: () => ({ input: "input", metadata: undefined }), @@ -111,32 +126,27 @@ describe("traceAsyncChannel current span binding", () => { extractMetrics: () => ({}), }); - try { - await testChannels.asyncCall.tracePromise( - async () => { - const parent = currentSpan(); - parent.startSpan({ name: "user-child" }).end(); - parent - .startSpanWithParents("user-multi-parent-child", [parent.spanId], { - name: "user-multi-parent-child", - }) - .end(); - parent - .startSpan( - withSpanInstrumentationName( - { name: "instrumentation-child" }, - INSTRUMENTATION_NAMES.OPENAI, - ), - ) - .end(); - return { ok: true as const }; - }, - { arguments: [{}] } as any, - ); - } finally { - unsubscribe(); - } - + await testChannels.provenance.tracePromise( + async () => { + const parent = currentSpan(); + parent.startSpan({ name: "user-child" }).end(); + parent + .startSpanWithParents("user-multi-parent-child", [parent.spanId], { + name: "user-multi-parent-child", + }) + .end(); + parent + .startSpan( + withSpanInstrumentationName( + { name: "instrumentation-child" }, + INSTRUMENTATION_NAMES.OPENAI, + ), + ) + .end(); + return { ok: true as const }; + }, + { arguments: [{}] } as any, + ); const spans = (await backgroundLogger.drain()) as Array< Record >; @@ -161,7 +171,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("does not create a span when shouldTrace returns false", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.skipped, { name: "channel-tracing-test", shouldTrace: ([params]) => !( @@ -181,21 +191,16 @@ describe("traceAsyncChannel current span binding", () => { const seenSpanIds: string[] = []; - try { - await testChannels.asyncCall.tracePromise( - async () => { - seenSpanIds.push(currentSpan().spanId); - await Promise.resolve(); - seenSpanIds.push(currentSpan().spanId); - - return { ok: true as const }; - }, - { arguments: [{ skip: true }] } as any, - ); - } finally { - unsubscribe(); - } + await testChannels.skipped.tracePromise( + async () => { + seenSpanIds.push(currentSpan().spanId); + await Promise.resolve(); + seenSpanIds.push(currentSpan().spanId); + return { ok: true as const }; + }, + { arguments: [{ skip: true }] } as any, + ); expect(seenSpanIds).toEqual(["", ""]); expect(currentSpan()).toBe(NOOP_SPAN); @@ -207,7 +212,7 @@ describe("traceAsyncChannel current span binding", () => { const consoleErrorSpy = vi .spyOn(console, "error") .mockImplementation(() => {}); - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.throwingPredicate, { name: "channel-tracing-test", shouldTrace: () => { throw new Error("predicate failed"); @@ -221,15 +226,10 @@ describe("traceAsyncChannel current span binding", () => { extractMetrics: () => ({}), }); - try { - await testChannels.asyncCall.tracePromise( - async () => ({ ok: true as const }), - { arguments: [{}] } as any, - ); - } finally { - unsubscribe(); - } - + await testChannels.throwingPredicate.tracePromise( + async () => ({ ok: true as const }), + { arguments: [{}] } as any, + ); expect(consoleErrorSpy).not.toHaveBeenCalled(); consoleErrorSpy.mockRestore(); @@ -238,7 +238,7 @@ describe("traceAsyncChannel current span binding", () => { }); it("skips auto instrumentation spans while suppression is active", async () => { - const unsubscribe = traceAsyncChannel(testChannels.asyncCall, { + traceAsyncChannel(testChannels.suppressed, { name: "channel-tracing-test", type: "function", extractInput: () => ({ @@ -249,23 +249,18 @@ describe("traceAsyncChannel current span binding", () => { extractMetrics: () => ({}), }); - try { - await runWithAutoInstrumentationSuppressed(() => - testChannels.asyncCall.tracePromise( - async () => { - expect(currentSpan()).toBe(NOOP_SPAN); - await Promise.resolve(); - expect(currentSpan()).toBe(NOOP_SPAN); - - return { ok: true as const }; - }, - { arguments: [{}] } as any, - ), - ); - } finally { - unsubscribe(); - } + await runWithAutoInstrumentationSuppressed(() => + testChannels.suppressed.tracePromise( + async () => { + expect(currentSpan()).toBe(NOOP_SPAN); + await Promise.resolve(); + expect(currentSpan()).toBe(NOOP_SPAN); + return { ok: true as const }; + }, + { arguments: [{}] } as any, + ), + ); const spans = await backgroundLogger.drain(); expect(spans).toHaveLength(0); }); @@ -280,7 +275,7 @@ describe("traceAsyncChannel current span binding", () => { throw new Error("logging failed"); }), } as unknown as Span; - const unsubscribe = traceStreamingChannel(testChannels.streamingCall, { + traceStreamingChannel(testChannels.streamingCleanup, { name: "streaming-channel-test", startSpan: () => child, type: "function", @@ -291,25 +286,20 @@ describe("traceAsyncChannel current span binding", () => { onError, }); - try { - await expect( - testChannels.streamingCall.tracePromise( - async () => ({ ok: true as const }), - { arguments: [{}] } as any, - ), - ).resolves.toEqual({ ok: true }); - await expect( - testChannels.streamingCall.tracePromise( - async () => { - throw new Error("call failed"); - }, - { arguments: [{}] } as any, - ), - ).rejects.toThrow("call failed"); - } finally { - unsubscribe(); - } - + await expect( + testChannels.streamingCleanup.tracePromise( + async () => ({ ok: true as const }), + { arguments: [{}] } as any, + ), + ).resolves.toEqual({ ok: true }); + await expect( + testChannels.streamingCleanup.tracePromise( + async () => { + throw new Error("call failed"); + }, + { arguments: [{}] } as any, + ), + ).rejects.toThrow("call failed"); expect(onComplete).toHaveBeenCalledTimes(1); expect(onError).toHaveBeenCalledTimes(1); expect(end).toHaveBeenCalledTimes(2); @@ -321,7 +311,7 @@ describe("traceAsyncChannel current span binding", () => { end: vi.fn(), log: vi.fn(), } as unknown as Span; - const unsubscribe = traceStreamingChannel(testChannels.streamingCall, { + traceStreamingChannel(testChannels.streamingCancellation, { name: "streaming-channel-test", startSpan: () => child, type: "function", @@ -337,17 +327,12 @@ describe("traceAsyncChannel current span binding", () => { }, }; - try { - const patched = await testChannels.streamingCall.tracePromise( - async () => stream as any, - { arguments: [{}] } as any, - ); - (patched as unknown as typeof stream).abort(); - await Promise.resolve(); - } finally { - unsubscribe(); - } - + const patched = await testChannels.streamingCancellation.tracePromise( + async () => stream as any, + { arguments: [{}] } as any, + ); + (patched as unknown as typeof stream).abort(); + await Promise.resolve(); const cancellationError = expect.objectContaining({ message: "Stream cancelled before completion", name: "AbortError", diff --git a/js/src/instrumentation/core/channel-tracing.ts b/js/src/instrumentation/core/channel-tracing.ts index a8ecc819c..1f3ff47d0 100644 --- a/js/src/instrumentation/core/channel-tracing.ts +++ b/js/src/instrumentation/core/channel-tracing.ts @@ -312,7 +312,7 @@ function bindCurrentSpanStoreToStart< }, channelName: string, instrumentationName: SpanInstrumentationName, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const startChannel = tracingChannel.start; const contextManager = state?.contextManager; @@ -325,7 +325,7 @@ function bindCurrentSpanStoreToStart< : undefined; if (!currentSpanStore || !startChannel) { - return undefined; + return; } startChannel.bindStore( @@ -347,10 +347,6 @@ function bindCurrentSpanStoreToStart< : currentSpanStore.getStore(); }, ); - - return () => { - startChannel.unbindStore(currentSpanStore); - }; } function logErrorAndEnd< @@ -442,13 +438,13 @@ function runStreamingErrorHook(args: { export function traceAsyncChannel( channel: TChannel, config: AsyncChannelSpanConfig, -): () => void { +): void { const tracingChannel = channel.tracingChannel() as IsoTracingChannel< ChannelMessage >; const states = new WeakMap(); const channelName = channel.channelName; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + bindCurrentSpanStoreToStart( tracingChannel, states, config, @@ -514,23 +510,18 @@ export function traceAsyncChannel( }; tracingChannel.subscribe(handlers); - - return () => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }; } export function traceStreamingChannel( channel: TChannel, config: StreamingChannelSpanConfig, -): () => void { +): void { const tracingChannel = channel.tracingChannel() as IsoTracingChannel< ChannelMessage >; const states = new WeakMap(); const channelName = channel.channelName; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + bindCurrentSpanStoreToStart( tracingChannel, states, config, @@ -790,23 +781,18 @@ export function traceStreamingChannel( }; tracingChannel.subscribe(handlers); - - return () => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }; } export function traceSyncStreamChannel( channel: TChannel, config: SyncStreamChannelSpanConfig, -): () => void { +): void { const tracingChannel = channel.tracingChannel() as IsoTracingChannel< ChannelMessage >; const states = new WeakMap(); const channelName = channel.channelName; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( + bindCurrentSpanStoreToStart( tracingChannel, states, config, @@ -939,19 +925,4 @@ export function traceSyncStreamChannel( }; tracingChannel.subscribe(handlers); - - return () => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }; -} - -export function unsubscribeAll( - unsubscribers: Array<() => void>, -): Array<() => void> { - for (const unsubscribe of unsubscribers) { - unsubscribe(); - } - - return []; } diff --git a/js/src/instrumentation/core/channel.ts b/js/src/instrumentation/core/channel.ts deleted file mode 100644 index 8cbf253d6..000000000 --- a/js/src/instrumentation/core/channel.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Utilities for instrumentation hook naming and management. - */ - -/** - * Channel naming convention: braintrust:{component}:{operation} - * - * Examples: - * - braintrust:openai:chat.completions.create - * - braintrust:anthropic:messages.create - * - braintrust:ai-sdk:generateText - */ - -/** - * Creates a standardized channel name. - * - * @param component - The SDK/library being instrumented (e.g., 'openai', 'anthropic') - * @param operation - The operation being traced (e.g., 'chat.completions.create') - * @returns The full channel name - */ -export function createChannelName( - component: string, - operation: string, -): string { - return `braintrust:${component}:${operation}`; -} - -/** - * Parses a channel name into its component parts. - * - * @param channelName - The full channel name - * @returns Object with component and operation, or null if invalid - */ -export function parseChannelName( - channelName: string, -): { component: string; operation: string } | null { - const match = channelName.match(/^braintrust:([^:]+):(.+)$/); - if (!match) { - return null; - } - return { - component: match[1], - operation: match[2], - }; -} - -/** - * Validates a channel name follows the expected convention. - * - * @param channelName - The channel name to validate - * @returns True if valid - */ -export function isValidChannelName(channelName: string): boolean { - return /^braintrust:[^:]+:.+$/.test(channelName); -} diff --git a/js/src/instrumentation/core/index.ts b/js/src/instrumentation/core/index.ts index 6e2d2d61c..edfe17a89 100644 --- a/js/src/instrumentation/core/index.ts +++ b/js/src/instrumentation/core/index.ts @@ -1,27 +1,3 @@ -/** - * Core utilities for building auto-instrumentation plugins. - * - * Provides BasePlugin class and channel utilities following the OpenTelemetry - * InstrumentationBase pattern - core infrastructure lives here, but individual - * instrumentations can be separate packages. - * - * Note: auto-instrumentation config types are exposed from the Braintrust - * bundler subpaths, such as `braintrust/vite`. - */ +/** Internal utilities shared by Braintrust instrumentation consumers. */ -export { BasePlugin } from "./plugin"; export { toLoggedError } from "./logging"; -export { - createChannelName, - parseChannelName, - isValidChannelName, -} from "./channel"; -export type { - BaseContext, - StartEvent, - EndEvent, - ErrorEvent, - AsyncStartEvent, - AsyncEndEvent, - ChannelHandlers, -} from "./types"; diff --git a/js/src/instrumentation/core/plugin.ts b/js/src/instrumentation/core/plugin.ts deleted file mode 100644 index ae09a43f1..000000000 --- a/js/src/instrumentation/core/plugin.ts +++ /dev/null @@ -1,505 +0,0 @@ -import iso from "../../isomorph"; -import type { IsoChannelHandlers } from "../../isomorph"; -import { isAsyncIterable, patchStreamIfNeeded } from "./stream-patcher"; -import type { StartEvent } from "./types"; -import { startSpan } from "../../logger"; -import type { Span } from "../../logger"; -import { getCurrentUnixTimestamp } from "../../util"; -import { - buildStartSpanArgs, - mergeInputMetadata, -} from "./channel-tracing-utils"; - -/** - * Base class for creating instrumentation plugins. - * - * Plugins subscribe to global instrumentation hook events and convert them - * into spans, logs, or other observability data. - */ -export abstract class BasePlugin { - protected enabled = false; - protected unsubscribers: Array<() => void> = []; - - /** - * Enables the plugin. Must be called before the plugin will receive events. - */ - enable(): void { - if (this.enabled) { - return; - } - this.enabled = true; - this.onEnable(); - } - - /** - * Disables the plugin. After this, the plugin will no longer receive events. - */ - disable(): void { - if (!this.enabled) { - return; - } - this.enabled = false; - this.onDisable(); - } - - /** - * Called when the plugin is enabled. - * Override this to set up subscriptions. - */ - protected abstract onEnable(): void; - - /** - * Called when the plugin is disabled. - * Override this to clean up subscriptions. - */ - protected abstract onDisable(): void; - - /** - * Helper to subscribe to a channel with raw handlers. - * - * @param channelName - The channel name to subscribe to - * @param handlers - Event handlers - */ - protected subscribe(channelName: string, handlers: IsoChannelHandlers): void { - const channel = iso.newTracingChannel(channelName); - channel.subscribe(handlers); - } - - /** - * Subscribe to a channel for async methods (non-streaming). - * Creates a span and logs input/output/metrics. - */ - protected subscribeToChannel( - channelName: string, - config: { - name: string; - type: string; - extractInput: (args: any[]) => { input: any; metadata: any }; - extractOutput: (result: any, endEvent?: any) => any; - extractMetadata?: (result: any, endEvent?: any) => any; - extractMetrics: ( - result: any, - startTime?: number, - endEvent?: any, - ) => Record; - }, - ): void { - const channel = iso.newTracingChannel(channelName); - - const spans = new WeakMap(); - - const handlers = { - start: (event: StartEvent) => { - const { name, spanAttributes, spanInfoMetadata } = buildStartSpanArgs( - config, - event, - ); - const span = startSpan({ - name, - spanAttributes, - }); - - const startTime = getCurrentUnixTimestamp(); - spans.set(event, { span, startTime }); - - try { - const { input, metadata } = config.extractInput(event.arguments); - span.log({ - input, - metadata: mergeInputMetadata(metadata, spanInfoMetadata), - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting input for ${channelName}:`, error); - } - }, - - asyncEnd: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span, startTime } = spanData; - - try { - const output = config.extractOutput(event.result, event); - const metrics = config.extractMetrics(event.result, startTime, event); - const metadata = config.extractMetadata?.(event.result, event); - - span.log({ - output, - ...(metadata !== undefined ? { metadata } : {}), - metrics, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting output for ${channelName}:`, error); - } finally { - span.end(); - spans.delete(event); - } - }, - - error: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span } = spanData; - - span.log({ - error: event.error.message, - }); - span.end(); - spans.delete(event); - }, - }; - - channel.subscribe(handlers); - - // Store unsubscribe function - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); - } - - /** - * Subscribe to a channel for async methods that may return streams. - * Handles both streaming and non-streaming responses. - */ - protected subscribeToStreamingChannel( - channelName: string, - config: { - name: string; - type: string; - extractInput: (args: any[]) => { input: any; metadata: any }; - extractOutput: (result: any, endEvent?: any) => any; - extractMetadata?: (result: any, endEvent?: any) => any; - extractMetrics: ( - result: any, - startTime?: number, - endEvent?: any, - ) => Record; - aggregateChunks?: ( - chunks: any[], - result?: any, - endEvent?: any, - ) => { - output: any; - metrics: Record; - metadata?: any; - }; - }, - ): void { - const channel = iso.newTracingChannel(channelName); - - const spans = new WeakMap(); - - const handlers = { - start: (event: StartEvent) => { - const { name, spanAttributes, spanInfoMetadata } = buildStartSpanArgs( - config, - event, - ); - const span = startSpan({ - name, - spanAttributes, - }); - - const startTime = getCurrentUnixTimestamp(); - spans.set(event, { span, startTime }); - - try { - const { input, metadata } = config.extractInput(event.arguments); - span.log({ - input, - metadata: mergeInputMetadata(metadata, spanInfoMetadata), - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting input for ${channelName}:`, error); - } - }, - - asyncEnd: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span, startTime } = spanData; - - // Check if result is a stream - if (isAsyncIterable(event.result)) { - let firstChunkTime: number | undefined; - - // Patch the stream to collect chunks - patchStreamIfNeeded(event.result, { - onChunk: () => { - if (firstChunkTime === undefined) { - firstChunkTime = getCurrentUnixTimestamp(); - } - }, - onComplete: (chunks: any[]) => { - try { - let output: any; - let metrics: Record; - let metadata: any; - - if (config.aggregateChunks) { - const aggregated = config.aggregateChunks( - chunks, - event.result, - event, - ); - output = aggregated.output; - metrics = aggregated.metrics; - metadata = aggregated.metadata; - } else { - output = config.extractOutput(chunks, event); - metrics = config.extractMetrics(chunks, startTime, event); - } - - // Add time_to_first_token if not already present - if ( - metrics.time_to_first_token === undefined && - firstChunkTime !== undefined - ) { - metrics.time_to_first_token = firstChunkTime - startTime; - } else if ( - metrics.time_to_first_token === undefined && - chunks.length > 0 - ) { - metrics.time_to_first_token = - getCurrentUnixTimestamp() - startTime; - } - - span.log({ - output, - ...(metadata !== undefined ? { metadata } : {}), - metrics, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Error extracting output for ${channelName}:`, - error, - ); - } finally { - span.end(); - } - }, - onError: (error: Error) => { - span.log({ - error: error.message, - }); - span.end(); - }, - }); - - // Don't delete the span from the map yet - it will be ended by the stream - } else { - // Non-streaming response - try { - const output = config.extractOutput(event.result, event); - const metadata = config.extractMetadata - ? config.extractMetadata(event.result, event) - : undefined; - const metrics = config.extractMetrics( - event.result, - startTime, - event, - ); - - span.log({ - output, - ...(metadata !== undefined ? { metadata } : {}), - metrics, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting output for ${channelName}:`, error); - } finally { - span.end(); - spans.delete(event); - } - } - }, - - error: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span } = spanData; - - span.log({ - error: event.error.message, - }); - span.end(); - spans.delete(event); - }, - }; - - channel.subscribe(handlers); - - // Store unsubscribe function - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); - } - - /** - * Subscribe to a channel for sync methods that return event-based streams. - * Used for methods like beta.chat.completions.stream() and responses.stream(). - */ - protected subscribeToSyncStreamChannel( - channelName: string, - config: { - name: string; - type: string; - extractInput: (args: any[]) => { input: any; metadata: any }; - extractFromEvent?: (event: any) => { - output?: any; - metrics?: Record; - metadata?: any; - }; - }, - ): void { - const channel = iso.newTracingChannel(channelName); - - const spans = new WeakMap(); - - const handlers = { - start: (event: StartEvent) => { - const { name, spanAttributes, spanInfoMetadata } = buildStartSpanArgs( - config, - event, - ); - const span = startSpan({ - name, - spanAttributes, - }); - - const startTime = getCurrentUnixTimestamp(); - spans.set(event, { span, startTime }); - - try { - const { input, metadata } = config.extractInput(event.arguments); - span.log({ - input, - metadata: mergeInputMetadata(metadata, spanInfoMetadata), - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error(`Error extracting input for ${channelName}:`, error); - } - }, - - end: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span, startTime } = spanData; - const stream = event.result; - - if (!stream || typeof stream.on !== "function") { - // Not a stream, just end the span - span.end(); - spans.delete(event); - return; - } - - let first = true; - - // Listen for stream events - stream.on("chunk", (chunk: any) => { - if (first) { - const now = getCurrentUnixTimestamp(); - span.log({ - metrics: { - time_to_first_token: now - startTime, - }, - }); - first = false; - } - }); - - stream.on("chatCompletion", (completion: any) => { - try { - span.log({ - output: completion.choices, - }); - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Error extracting chatCompletion for ${channelName}:`, - error, - ); - } - }); - - stream.on("event", (streamEvent: any) => { - if (config.extractFromEvent) { - try { - if (first) { - const now = getCurrentUnixTimestamp(); - span.log({ - metrics: { - time_to_first_token: now - startTime, - }, - }); - first = false; - } - - const extracted = config.extractFromEvent(streamEvent); - if (extracted && Object.keys(extracted).length > 0) { - span.log(extracted); - } - } catch (error) { - // eslint-disable-next-line no-restricted-properties -- preserving intentional console usage. - console.error( - `Error extracting event for ${channelName}:`, - error, - ); - } - } - }); - - stream.on("end", () => { - span.end(); - spans.delete(event); - }); - - // Don't delete the span from the map - it will be deleted when the stream ends - }, - - error: (event: any) => { - const spanData = spans.get(event); - if (!spanData) { - return; - } - - const { span } = spanData; - - span.log({ - error: event.error.message, - }); - span.end(); - spans.delete(event); - }, - }; - - channel.subscribe(handlers); - - // Store unsubscribe function - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); - } -} diff --git a/js/src/instrumentation/core/types.ts b/js/src/instrumentation/core/types.ts index dc2434dc3..0181f3196 100644 --- a/js/src/instrumentation/core/types.ts +++ b/js/src/instrumentation/core/types.ts @@ -27,7 +27,7 @@ export type SpanInfoCarrier< /** * Base context object shared across all events in a trace. */ -export interface BaseContext { +interface BaseContext { /** * Unique identifier for this trace. * Can be used to correlate start/end/error events. @@ -52,42 +52,6 @@ export interface StartEvent extends BaseContext { arguments: TInput[]; } -/** - * Event emitted after the synchronous portion completes. - * For async functions, this fires when the promise is returned (not settled). - */ -export interface EndEvent extends BaseContext { - /** - * The result of the synchronous portion. - * For async functions, this is the promise (not the resolved value). - */ - result: TResult; - - /** - * Arguments passed to the function (also available in StartEvent). - */ - arguments?: unknown[]; -} - -/** - * Event emitted when a function throws or a promise rejects. - */ -export interface ErrorEvent extends BaseContext { - /** - * The error that was thrown or the rejection reason. - */ - error: Error; - - /** - * Arguments passed to the function (also available in StartEvent). - */ - arguments?: unknown[]; -} - -/** - * Event emitted when a promise begins to settle. - * This fires after the synchronous portion and when the async continuation starts. - */ export interface TypedStartEvent< TArguments extends EventArguments = unknown[], > extends BaseContext { @@ -112,17 +76,6 @@ export interface TypedErrorEvent< arguments?: [...TArguments, ...any[]]; } -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface AsyncStartEvent extends StartEvent {} - -/** - * Event emitted when a promise finishes settling. - * This fires BEFORE control returns to user code after await. - * This is where you should extract output data and finalize spans. - */ -// eslint-disable-next-line @typescript-eslint/no-empty-object-type -export interface AsyncEndEvent extends EndEvent {} - export type StartEventWith< TArguments extends EventArguments = unknown[], TExtra extends object = Record, @@ -144,43 +97,3 @@ export type ErrorEventWith< TArguments extends EventArguments = unknown[], TExtra extends object = Record, > = TypedErrorEvent & TExtra; - -/** - * Subscription handlers for a tracing-compatible global hook. - * - * Common usage pattern: - * - Use start to create spans and extract input - * - Use asyncEnd to extract output and finalize spans - * - Use error to handle failures - */ -export interface ChannelHandlers { - /** - * Called before the synchronous portion of a function executes. - * Use this to create spans and extract input data. - */ - start?: (event: StartEvent) => void; - - /** - * Called after the synchronous portion completes (promise returned). - * Usually not needed for typical instrumentation. - */ - end?: (event: EndEvent) => void; - - /** - * Called when a promise begins to settle. - * Usually not needed for typical instrumentation. - */ - asyncStart?: (event: AsyncStartEvent) => void; - - /** - * Called when a promise finishes settling, before user code continues. - * Use this to extract output, patch streams, and finalize spans. - */ - asyncEnd?: (event: AsyncEndEvent) => void; - - /** - * Called when a function throws or promise rejects. - * Use this to log errors and clean up spans. - */ - error?: (event: ErrorEvent) => void; -} diff --git a/js/src/instrumentation/index.test.ts b/js/src/instrumentation/index.test.ts new file mode 100644 index 000000000..fd5442745 --- /dev/null +++ b/js/src/instrumentation/index.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import * as instrumentation from "./index"; + +describe("instrumentation public API", () => { + it("exposes only the curated instrumentation APIs", () => { + expect(Object.keys(instrumentation).sort()).toEqual([ + "OpenAIAgentsTraceProcessor", + "braintrustEveInstrumentation", + "braintrustFlueInstrumentation", + "configureInstrumentation", + ]); + }); +}); diff --git a/js/src/instrumentation/index.ts b/js/src/instrumentation/index.ts index 338d62abd..e148f2dc5 100644 --- a/js/src/instrumentation/index.ts +++ b/js/src/instrumentation/index.ts @@ -1,48 +1,16 @@ /** * Instrumentation APIs for auto-instrumentation. * - * This module provides the core plugin infrastructure for converting global - * instrumentation hook events into Braintrust spans. - * - * Following the OpenTelemetry pattern, BasePlugin (like InstrumentationBase) - * lives in the core SDK, while individual instrumentation implementations - * can be separate packages. - * * For auto-instrumentation config types, import InstrumentationConfig from the * relevant Braintrust bundler subpath such as `braintrust/vite`. * * @module instrumentation */ -export { BasePlugin } from "./core"; -export { BraintrustPlugin } from "./braintrust-plugin"; -export type { BraintrustPluginConfig } from "./braintrust-plugin"; -export { OpenAIAgentsTraceProcessor } from "./plugins/openai-agents-trace-processor"; -export type { OpenAIAgentsTraceProcessorOptions } from "./plugins/openai-agents-trace-processor"; -export { - braintrustFlueInstrumentation, - braintrustFlueObserver, -} from "./plugins/flue-plugin"; -export { - braintrustEveHook, - braintrustEveInstrumentation, -} from "./plugins/eve-plugin"; - -// Re-export core types for external instrumentation packages -export type { - BaseContext, - StartEvent, - EndEvent, - ErrorEvent, - AsyncStartEvent, - AsyncEndEvent, - ChannelHandlers, -} from "./core"; -export { - createChannelName, - parseChannelName, - isValidChannelName, -} from "./core"; +export { OpenAIAgentsTraceProcessor } from "./providers/openai-agents-trace-processor"; +export type { OpenAIAgentsTraceProcessorOptions } from "./providers/openai-agents-trace-processor"; +export { braintrustFlueInstrumentation } from "./providers/flue-instrumentation"; +export { braintrustEveInstrumentation } from "./providers/eve-instrumentation"; // Configuration API export { configureInstrumentation } from "./registry"; diff --git a/js/src/instrumentation/instrumentation-consumers.test.ts b/js/src/instrumentation/instrumentation-consumers.test.ts new file mode 100644 index 000000000..ce4837bb5 --- /dev/null +++ b/js/src/instrumentation/instrumentation-consumers.test.ts @@ -0,0 +1,215 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { InstrumentationIntegrationsConfig } from "./config"; + +const registrations = vi.hoisted(() => ({ + aiSDK: vi.fn(), + anthropic: vi.fn(), + bedrockRuntime: vi.fn(), + claudeAgentSDK: vi.fn(), + cloudflareAgents: vi.fn(), + cloudflareAIChat: vi.fn(), + cloudflareThink: vi.fn(), + cohere: vi.fn(), + cursorSDK: vi.fn(), + genkit: vi.fn(), + gitHubCopilot: vi.fn(), + googleADK: vi.fn(), + googleGenAI: vi.fn(), + groq: vi.fn(), + huggingFace: vi.fn(), + huggingFaceTransformers: vi.fn(), + langChain: vi.fn(), + langSmith: vi.fn(), + mistral: vi.fn(), + ollama: vi.fn(), + openAI: vi.fn(), + openAIAgents: vi.fn(), + openAICodex: vi.fn(), + openRouter: vi.fn(), + openRouterAgent: vi.fn(), + piCodingAgent: vi.fn(), + strandsAgentSDK: vi.fn(), + voyageAI: vi.fn(), +})); + +vi.mock("./providers/ai-sdk-instrumentation", () => ({ + registerAISDKInstrumentation: registrations.aiSDK, +})); +vi.mock("./providers/anthropic-instrumentation", () => ({ + registerAnthropicInstrumentation: registrations.anthropic, +})); +vi.mock("./providers/bedrock-runtime-instrumentation", () => ({ + registerBedrockRuntimeInstrumentation: registrations.bedrockRuntime, +})); +vi.mock("./providers/claude-agent-sdk-instrumentation", () => ({ + registerClaudeAgentSDKInstrumentation: registrations.claudeAgentSDK, +})); +vi.mock("./providers/cloudflare-agents-instrumentation", () => ({ + registerCloudflareAgentsInstrumentation: registrations.cloudflareAgents, +})); +vi.mock("./providers/cloudflare-ai-chat-consumer", () => ({ + registerCloudflareAIChatInstrumentation: registrations.cloudflareAIChat, +})); +vi.mock("./providers/cloudflare-think-instrumentation", () => ({ + registerCloudflareThinkInstrumentation: registrations.cloudflareThink, +})); +vi.mock("./providers/cohere-instrumentation", () => ({ + registerCohereInstrumentation: registrations.cohere, +})); +vi.mock("./providers/cursor-sdk-instrumentation", () => ({ + registerCursorSDKInstrumentation: registrations.cursorSDK, +})); +vi.mock("./providers/genkit-instrumentation", () => ({ + registerGenkitInstrumentation: registrations.genkit, +})); +vi.mock("./providers/github-copilot-instrumentation", () => ({ + registerGitHubCopilotInstrumentation: registrations.gitHubCopilot, +})); +vi.mock("./providers/google-adk-instrumentation", () => ({ + registerGoogleADKInstrumentation: registrations.googleADK, +})); +vi.mock("./providers/google-genai-instrumentation", () => ({ + registerGoogleGenAIInstrumentation: registrations.googleGenAI, +})); +vi.mock("./providers/groq-instrumentation", () => ({ + registerGroqInstrumentation: registrations.groq, +})); +vi.mock("./providers/huggingface-instrumentation", () => ({ + registerHuggingFaceInstrumentation: registrations.huggingFace, +})); +vi.mock("./providers/huggingface-transformers-instrumentation", () => ({ + registerHuggingFaceTransformersInstrumentation: + registrations.huggingFaceTransformers, +})); +vi.mock("./providers/langchain-instrumentation", () => ({ + registerLangChainInstrumentation: registrations.langChain, +})); +vi.mock("./providers/langsmith-instrumentation", () => ({ + registerLangSmithInstrumentation: registrations.langSmith, +})); +vi.mock("./providers/mistral-instrumentation", () => ({ + registerMistralInstrumentation: registrations.mistral, +})); +vi.mock("./providers/ollama-instrumentation", () => ({ + registerOllamaInstrumentation: registrations.ollama, +})); +vi.mock("./providers/openai-instrumentation", () => ({ + registerOpenAIInstrumentation: registrations.openAI, +})); +vi.mock("./providers/openai-agents-instrumentation", () => ({ + registerOpenAIAgentsInstrumentation: registrations.openAIAgents, +})); +vi.mock("./providers/openai-codex-instrumentation", () => ({ + registerOpenAICodexInstrumentation: registrations.openAICodex, +})); +vi.mock("./providers/openrouter-instrumentation", () => ({ + registerOpenRouterInstrumentation: registrations.openRouter, +})); +vi.mock("./providers/openrouter-agent-instrumentation", () => ({ + registerOpenRouterAgentInstrumentation: registrations.openRouterAgent, +})); +vi.mock("./providers/pi-coding-agent-instrumentation", () => ({ + registerPiCodingAgentInstrumentation: registrations.piCodingAgent, +})); +vi.mock("./providers/strands-agent-sdk-instrumentation", () => ({ + registerStrandsAgentSDKInstrumentation: registrations.strandsAgentSDK, +})); +vi.mock("./providers/voyageai-instrumentation", () => ({ + registerVoyageAIInstrumentation: registrations.voyageAI, +})); + +import { registerInstrumentationConsumers } from "./instrumentation-consumers"; + +const integrationCases: Array< + [ + keyof InstrumentationIntegrationsConfig, + Array<(typeof registrations)[keyof typeof registrations]>, + ] +> = [ + ["aisdk", [registrations.aiSDK]], + ["anthropic", [registrations.anthropic]], + ["bedrock", [registrations.bedrockRuntime]], + ["claudeAgentSDK", [registrations.claudeAgentSDK]], + ["cloudflareAgents", [registrations.cloudflareAgents]], + ["cloudflareAIChat", [registrations.cloudflareAIChat]], + ["cloudflareThink", [registrations.cloudflareThink]], + ["cohere", [registrations.cohere]], + ["cursorSDK", [registrations.cursorSDK]], + ["genkit", [registrations.genkit]], + ["gitHubCopilot", [registrations.gitHubCopilot]], + ["googleADK", [registrations.googleADK]], + ["googleGenAI", [registrations.googleGenAI]], + ["groq", [registrations.groq]], + [ + "huggingface", + [registrations.huggingFace, registrations.huggingFaceTransformers], + ], + ["langchain", [registrations.langChain]], + ["langsmith", [registrations.langSmith]], + ["mistral", [registrations.mistral]], + ["ollama", [registrations.ollama]], + ["openai", [registrations.openAI]], + ["openAIAgents", [registrations.openAIAgents]], + ["openaiCodexSDK", [registrations.openAICodex]], + ["openrouter", [registrations.openRouter]], + ["openrouterAgent", [registrations.openRouterAgent]], + ["piCodingAgent", [registrations.piCodingAgent]], + ["strandsAgentSDK", [registrations.strandsAgentSDK]], + ["voyageai", [registrations.voyageAI]], +]; + +describe("registerInstrumentationConsumers", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("registers every instrumentation consumer by default", () => { + registerInstrumentationConsumers(); + + for (const register of Object.values(registrations)) { + expect(register).toHaveBeenCalledTimes(1); + } + expect(registrations.langSmith).toHaveBeenCalledWith({ + skipLangChainRuns: true, + }); + }); + + it.each(integrationCases)( + "does not register the %s integration when disabled", + (integration, disabledRegistrations) => { + registerInstrumentationConsumers({ + integrations: { [integration]: false }, + }); + + for (const register of disabledRegistrations) { + expect(register).not.toHaveBeenCalled(); + } + }, + ); + + it.each([ + ["vercel", registrations.aiSDK], + ["google", registrations.googleGenAI], + ["cursor", registrations.cursorSDK], + ["awsBedrock", registrations.bedrockRuntime], + ["awsBedrockRuntime", registrations.bedrockRuntime], + ["langgraph", registrations.langChain], + ] as const)( + "honors the legacy or related %s disable flag", + (integration, register) => { + registerInstrumentationConsumers({ + integrations: { [integration]: false }, + }); + + expect(register).not.toHaveBeenCalled(); + }, + ); + + it("tells LangSmith not to suppress LangChain runs when LangChain is disabled", () => { + registerInstrumentationConsumers({ integrations: { langchain: false } }); + + expect(registrations.langSmith).toHaveBeenCalledWith({ + skipLangChainRuns: false, + }); + }); +}); diff --git a/js/src/instrumentation/instrumentation-consumers.ts b/js/src/instrumentation/instrumentation-consumers.ts new file mode 100644 index 000000000..ed32522a9 --- /dev/null +++ b/js/src/instrumentation/instrumentation-consumers.ts @@ -0,0 +1,159 @@ +import { registerOpenAIInstrumentation } from "./providers/openai-instrumentation"; +import { registerOpenAICodexInstrumentation } from "./providers/openai-codex-instrumentation"; +import { registerAnthropicInstrumentation } from "./providers/anthropic-instrumentation"; +import { registerAISDKInstrumentation } from "./providers/ai-sdk-instrumentation"; +import { registerClaudeAgentSDKInstrumentation } from "./providers/claude-agent-sdk-instrumentation"; +import { registerCloudflareThinkInstrumentation } from "./providers/cloudflare-think-instrumentation"; +import { registerCursorSDKInstrumentation } from "./providers/cursor-sdk-instrumentation"; +import { registerOpenAIAgentsInstrumentation } from "./providers/openai-agents-instrumentation"; +import { registerGoogleGenAIInstrumentation } from "./providers/google-genai-instrumentation"; +import { registerHuggingFaceInstrumentation } from "./providers/huggingface-instrumentation"; +import { registerHuggingFaceTransformersInstrumentation } from "./providers/huggingface-transformers-instrumentation"; +import { registerOpenRouterAgentInstrumentation } from "./providers/openrouter-agent-instrumentation"; +import { registerOpenRouterInstrumentation } from "./providers/openrouter-instrumentation"; +import { registerMistralInstrumentation } from "./providers/mistral-instrumentation"; +import { registerOllamaInstrumentation } from "./providers/ollama-instrumentation"; +import { registerGoogleADKInstrumentation } from "./providers/google-adk-instrumentation"; +import { registerCohereInstrumentation } from "./providers/cohere-instrumentation"; +import { registerGroqInstrumentation } from "./providers/groq-instrumentation"; +import { registerBedrockRuntimeInstrumentation } from "./providers/bedrock-runtime-instrumentation"; +import { registerGenkitInstrumentation } from "./providers/genkit-instrumentation"; +import { registerGitHubCopilotInstrumentation } from "./providers/github-copilot-instrumentation"; +import { registerLangChainInstrumentation } from "./providers/langchain-instrumentation"; +import { registerLangSmithInstrumentation } from "./providers/langsmith-instrumentation"; +import { registerPiCodingAgentInstrumentation } from "./providers/pi-coding-agent-instrumentation"; +import { registerStrandsAgentSDKInstrumentation } from "./providers/strands-agent-sdk-instrumentation"; +import { registerVoyageAIInstrumentation } from "./providers/voyageai-instrumentation"; +import { registerCloudflareAIChatInstrumentation } from "./providers/cloudflare-ai-chat-consumer"; +import { registerCloudflareAgentsInstrumentation } from "./providers/cloudflare-agents-instrumentation"; +import type { InstrumentationConfig } from "./config"; + +/** Registers the configured process-lifetime instrumentation consumers. */ +export function registerInstrumentationConsumers( + config: InstrumentationConfig = {}, +): void { + const integrations = config.integrations ?? {}; + + if (integrations.openai !== false) { + registerOpenAIInstrumentation(); + } + + if (integrations.openaiCodexSDK !== false) { + registerOpenAICodexInstrumentation(); + } + + if (integrations.anthropic !== false) { + registerAnthropicInstrumentation(); + } + + // Support both 'aisdk' and legacy 'vercel' config keys + if (integrations.aisdk !== false && integrations.vercel !== false) { + registerAISDKInstrumentation(); + } + + if (integrations.claudeAgentSDK !== false) { + registerClaudeAgentSDKInstrumentation(); + } + + if (integrations.cloudflareThink !== false) { + registerCloudflareThinkInstrumentation(); + } + + if (integrations.cursorSDK !== false && integrations.cursor !== false) { + registerCursorSDKInstrumentation(); + } + + if (integrations.openAIAgents !== false) { + registerOpenAIAgentsInstrumentation(); + } + + // Support both 'googleGenAI' and legacy 'google' config keys + if (integrations.googleGenAI !== false && integrations.google !== false) { + registerGoogleGenAIInstrumentation(); + } + + if (integrations.huggingface !== false) { + registerHuggingFaceInstrumentation(); + registerHuggingFaceTransformersInstrumentation(); + } + + if (integrations.openrouter !== false) { + registerOpenRouterInstrumentation(); + } + + if (integrations.openrouterAgent !== false) { + registerOpenRouterAgentInstrumentation(); + } + + if (integrations.mistral !== false) { + registerMistralInstrumentation(); + } + + if (integrations.ollama !== false) { + registerOllamaInstrumentation(); + } + + if (integrations.googleADK !== false) { + registerGoogleADKInstrumentation(); + } + + if (integrations.cohere !== false) { + registerCohereInstrumentation(); + } + + if (integrations.voyageai !== false) { + registerVoyageAIInstrumentation(); + } + + if (integrations.groq !== false) { + registerGroqInstrumentation(); + } + + if ( + integrations.bedrock !== false && + integrations.awsBedrock !== false && + integrations.awsBedrockRuntime !== false + ) { + registerBedrockRuntimeInstrumentation(); + } + + if (integrations.genkit !== false) { + registerGenkitInstrumentation(); + } + + if (integrations.gitHubCopilot !== false) { + registerGitHubCopilotInstrumentation(); + } + + if (integrations.piCodingAgent !== false) { + registerPiCodingAgentInstrumentation(); + } + + if (integrations.strandsAgentSDK !== false) { + registerStrandsAgentSDKInstrumentation(); + } + + if (integrations.cloudflareAIChat !== false) { + registerCloudflareAIChatInstrumentation(); + } + + if (integrations.cloudflareAgents !== false) { + registerCloudflareAgentsInstrumentation(); + } + + if (integrations.langchain !== false && integrations.langgraph !== false) { + registerLangChainInstrumentation(); + } + + if (integrations.langsmith !== false) { + registerLangSmithInstrumentation({ + skipLangChainRuns: integrations.langchain !== false, + }); + } + + // Mastra is intentionally not wired here: `@mastra/core` ships its own + // ObservabilityExporter contract, and `BraintrustObservabilityExporter` + // (wrappers/mastra.ts) is auto-installed by the loader patch in + // `auto-instrumentations/loader/mastra-observability-patch.ts` rather than + // by an instrumentation consumer / global hook subscription. +} diff --git a/js/src/instrumentation/plugins/cloudflare-agents-plugin.ts b/js/src/instrumentation/plugins/cloudflare-agents-plugin.ts deleted file mode 100644 index d9813f419..000000000 --- a/js/src/instrumentation/plugins/cloudflare-agents-plugin.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { debugLogger } from "../../debug-logger"; -import type { IsoChannelHandlers } from "../../isomorph"; -import { _internalStartSpanWithContext } from "../../logger"; -import type { Span } from "../../logger"; -import { - INSTRUMENTATION_NAMES, - withSpanInstrumentationName, -} from "../../span-origin"; -import { SpanTypeAttribute } from "../../../util/index"; -import { BasePlugin } from "../core"; -import type { ChannelMessage } from "../core/channel-definitions"; -import { cloudflareAgentsChannels } from "./cloudflare-agents-channels"; - -const CLOUDFLARE_WORKERS_CONTEXT = { - span_origin: { - environment: { type: "server", name: "cloudflare_workers" }, - }, -}; - -export class CloudflareAgentsPlugin extends BasePlugin { - protected onEnable(): void { - const channel = cloudflareAgentsChannels.runAgentTool.tracingChannel(); - const spans = new WeakMap(); - const handlers: IsoChannelHandlers< - ChannelMessage - > = { - start: (event) => { - try { - const agentClass = event.arguments[0]; - const options = event.arguments[1]; - if (ownValue(options, "detached")) { - return; - } - - const name = ownValue(agentClass, "name"); - if (typeof name !== "string" || name.length === 0) { - debugLogger.warn( - "Skipping Cloudflare Agents runAgentTool span because the child agent class has no name.", - ); - return; - } - - const span = _internalStartSpanWithContext( - withSpanInstrumentationName( - { - name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - event: { - input: ownValue(options, "input"), - }, - }, - INSTRUMENTATION_NAMES.CLOUDFLARE_AGENTS, - ), - CLOUDFLARE_WORKERS_CONTEXT, - ); - spans.set(event, span); - } catch (error) { - logInstrumentationError("start", error); - } - }, - asyncEnd: (event) => { - const span = spans.get(event); - if (!span) { - return; - } - spans.delete(event); - - try { - const status = ownValue(event.result, "status"); - if (status === "completed") { - span.log({ output: ownValue(event.result, "output") }); - } else { - const error = ownValue(event.result, "error"); - if (typeof error === "string") { - span.log({ error }); - } - } - } catch (error) { - logInstrumentationError("completion", error); - } finally { - safelyEndSpan(span); - } - }, - error: (event) => { - const span = spans.get(event); - if (!span) { - return; - } - spans.delete(event); - - try { - span.log({ error: event.error }); - } catch (error) { - logInstrumentationError("rejection", error); - } finally { - safelyEndSpan(span); - } - }, - }; - - channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); - } - - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } -} - -function ownValue(value: unknown, key: PropertyKey): unknown { - if (!isObjectLike(value)) { - return undefined; - } - const descriptor = Object.getOwnPropertyDescriptor(value, key); - return descriptor && "value" in descriptor ? descriptor.value : undefined; -} - -function isObjectLike(value: unknown): value is object { - return ( - (typeof value === "object" && value !== null) || typeof value === "function" - ); -} - -function safelyEndSpan(span: Span): void { - try { - span.end(); - } catch (error) { - logInstrumentationError("span end", error); - } -} - -function logInstrumentationError(operation: string, error: unknown): void { - debugLogger.error( - `Failed to process Cloudflare Agents ${operation} instrumentation:`, - error, - ); -} diff --git a/js/src/instrumentation/plugins/eve-plugin.test.ts b/js/src/instrumentation/plugins/eve-plugin.test.ts deleted file mode 100644 index bf4323b71..000000000 --- a/js/src/instrumentation/plugins/eve-plugin.test.ts +++ /dev/null @@ -1,2271 +0,0 @@ -import { - afterEach, - beforeAll, - beforeEach, - describe, - expect, - it, - vi, -} from "vitest"; -import { createHash } from "node:crypto"; -import * as braintrustExports from "../../exports"; -import { configureNode } from "../../node/config"; -import { - _exportsForTestingOnly, - initLogger, - startSpan, - withCurrent, -} from "../../logger"; -import * as instrumentationExports from "../index"; -import { braintrustEveHook, braintrustEveInstrumentation } from "./eve-plugin"; -import type { - EveHandleMessageStreamEvent, - EveHookContext, -} from "../../vendor-sdk-types/eve"; -import { mergeRowBatch } from "../../../util/index"; - -function deterministicEveIdForTest(...parts: string[]): string { - return createHash("sha256") - .update(parts.map((part) => `${part.length}:${part}`).join("\0")) - .digest("hex") - .slice(0, 32) - .replace(/^(.{8})(.{4})(.{4})(.{4})(.{12})$/, "$1-$2-$3-$4-$5"); -} - -function createFakeDefineState() { - const values = new Map(); - return { - defineState(name: string, initial: () => T) { - return { - get: () => (values.has(name) ? (values.get(name) as T) : initial()), - update: (fn: (current: T) => T) => { - values.set( - name, - fn(values.has(name) ? (values.get(name) as T) : initial()), - ); - }, - }; - }, - values, - }; -} - -try { - configureNode(); -} catch { - // Best-effort initialization for test environments. -} - -describe("braintrustEveHook", () => { - let backgroundLogger: ReturnType< - typeof _exportsForTestingOnly.useTestBackgroundLogger - >; - let defineState: ReturnType["defineState"]; - - beforeAll(async () => { - await _exportsForTestingOnly.simulateLoginForTests(); - }); - - beforeEach(() => { - defineState = createFakeDefineState().defineState; - backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - initLogger({ - projectName: "eve-plugin.test.ts", - projectId: "test-project-id", - }); - }); - - afterEach(() => { - vi.restoreAllMocks(); - _exportsForTestingOnly.clearTestBackgroundLogger(); - }); - - it("returns an Eve hook definition", () => { - const hook = braintrustEveHook({ defineState }); - - expect(Object.keys(hook)).toEqual(["events"]); - expect(typeof hook.events?.["*"]).toBe("function"); - }); - - it("returns an Eve instrumentation definition", () => { - const setup = vi.fn(); - const instrumentation = braintrustEveInstrumentation({ - defineState, - setup, - }); - - expect(instrumentation).toMatchObject({ - recordInputs: false, - recordOutputs: false, - setup, - }); - expect(typeof instrumentation.events?.["step.started"]).toBe("function"); - }); - - it("requires Eve's defineState API", () => { - expect(() => braintrustEveHook(undefined as never)).toThrow(); - expect(() => braintrustEveInstrumentation(undefined as never)).toThrow(); - }); - - it("exports Eve APIs from root and instrumentation entrypoints", () => { - expect(braintrustExports.braintrustEveHook).toBe(braintrustEveHook); - expect(braintrustExports.braintrustEveInstrumentation).toBe( - braintrustEveInstrumentation, - ); - expect(instrumentationExports.braintrustEveHook).toBe(braintrustEveHook); - expect(instrumentationExports.braintrustEveInstrumentation).toBe( - braintrustEveInstrumentation, - ); - }); - - it("captures stable model input without provider options", async () => { - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - const instrumentation = braintrustEveInstrumentation({ defineState }); - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-captured-input" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const modelInput = { - instructions: [ - { - content: "Answer with the relevant Eve instrumentation detail.", - providerOptions: { - openai: { reasoningEncryptedContent: "system-provider-secret" }, - }, - role: "system", - }, - { - content: "Keep the answer concise.", - role: "system", - }, - ], - messages: [ - { - content: [ - { - providerOptions: { - openai: { - itemId: "reasoning-item", - reasoningEncryptedContent: "must-not-be-logged", - }, - }, - text: "Readable reasoning", - type: "reasoning", - }, - { - input: { - providerOptions: { application: "tool-input" }, - reasoningEncryptedContent: "legitimate-tool-input", - }, - providerOptions: { - openai: { reasoningEncryptedContent: "tool-provider-secret" }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-call", - }, - { - output: { - providerOptions: { - openai: { - reasoningEncryptedContent: "output-provider-secret", - }, - }, - type: "json", - value: { - reasoningEncryptedContent: "legitimate-tool-output", - }, - }, - providerOptions: { - openai: { - reasoningEncryptedContent: "result-provider-secret", - }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-result", - }, - ], - providerOptions: { - openai: { reasoningEncryptedContent: "message-provider-secret" }, - }, - role: "assistant", - }, - ], - } as const; - - await emit({ - data: { - runtime: { - agentId: "agent-captured-input", - eveVersion: "0.22.1", - modelId: "dynamic:anthropic/claude-sonnet-5", - }, - }, - type: "session.started", - }); - await emit({ - data: { sequence: 0, turnId: "turn-captured-input" }, - type: "turn.started", - }); - instrumentation.events?.["step.started"]?.({ - modelInput, - session: { id: "session-captured-input" }, - step: { index: 0 }, - turn: { id: "turn-captured-input", sequence: 0 }, - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-captured-input" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Capture the model input.", - sequence: 0, - stepIndex: 0, - turnId: "turn-captured-input", - }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-captured-input", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-captured-input" }, - type: "turn.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const step = spans.find( - (span) => span.span_attributes?.name === "eve.step", - ); - const session = spans.find( - (span) => span.span_attributes?.name === "eve.session", - ); - const turn = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - for (const span of [turn, step]) { - expect(span?.context?.span_origin).toMatchObject({ - instrumentation: { name: "eve" }, - }); - } - expect(step?.input).toEqual([ - { - content: "Answer with the relevant Eve instrumentation detail.", - role: "system", - }, - { - content: "Keep the answer concise.", - role: "system", - }, - { - content: [ - { - text: "Readable reasoning", - type: "reasoning", - }, - { - input: { - providerOptions: { application: "tool-input" }, - reasoningEncryptedContent: "legitimate-tool-input", - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-call", - }, - { - output: { - type: "json", - value: { - reasoningEncryptedContent: "legitimate-tool-output", - }, - }, - toolCallId: "call-1", - toolName: "lookup", - type: "tool-result", - }, - ], - role: "assistant", - }, - ]); - expect(JSON.stringify(step?.input)).not.toContain("provider-secret"); - expect(JSON.stringify(step?.input)).not.toContain("must-not-be-logged"); - expect(step?.metadata).toEqual({ - "eve.session_id": "session-captured-input", - }); - expect(session).toBeUndefined(); - expect(turn?.metadata).toEqual({ - "eve.session_id": "session-captured-input", - }); - expect(step?.metadata).not.toHaveProperty("model"); - expect(step?.metadata).not.toHaveProperty("provider"); - expect(turn?.metadata).not.toHaveProperty("model"); - expect(turn?.metadata).not.toHaveProperty("provider"); - expect(fakeEve.values.get("braintrust.eve.tracing")).toMatchObject({ - llmInputs: [], - }); - }); - - it("skips missing or malformed Eve instrumentation state without throwing", async () => { - const instrumentation = braintrustEveInstrumentation({ defineState }); - expect(() => - instrumentation.events?.["step.started"]?.({ bad: true } as never), - ).not.toThrow(); - - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - fakeEve.values.set("braintrust.eve.tracing", { - llmInputs: [{ input: { content: "not an array" }, key: "bad" }], - }); - expect(() => - instrumentation.events?.["step.started"]?.({ - modelInput: { - messages: [{ content: "hello", role: "user" }], - }, - session: { id: "session-malformed-state" }, - step: { index: 0 }, - turn: { id: "turn-malformed-state", sequence: 0 }, - }), - ).not.toThrow(); - - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { session: { id: "session-no-state" } }; - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-no-state" }, - type: "turn.started", - }, - ctx, - ); - await wildcard?.( - { - data: { sequence: 0, stepIndex: 0, turnId: "turn-no-state" }, - type: "step.started", - }, - ctx, - ); - await wildcard?.( - { - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-no-state", - }, - type: "step.completed", - }, - ctx, - ); - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-no-state" }, - type: "turn.completed", - }, - ctx, - ); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const step = spans.find( - (span) => span.span_attributes?.name === "eve.step", - ); - expect(step?.input).toBeUndefined(); - }); - - it("bounds pre-existing durable trace state", async () => { - const fakeEve = createFakeDefineState(); - const oversizedEntryCount = 10_001; - fakeEve.values.set("braintrust.eve.tracing", { - llmInputs: [], - metadata: {}, - spanReferences: Array.from( - { length: oversizedEntryCount }, - (_, index) => ({ - exported: `exported-${index}`, - rootSpanId: `root-${index}`, - rowId: `row-${index}`, - spanId: `span-${index}`, - }), - ), - stepStarts: Array.from({ length: oversizedEntryCount }, (_, index) => ({ - open: false, - ordinal: index, - stepIndex: index, - turnId: `turn-${index}`, - })), - }); - const wildcard = braintrustEveHook({ - defineState: fakeEve.defineState, - }).events?.["*"]; - - await wildcard?.( - { - data: { - runtime: { - agentId: "agent-bounded-state", - eveVersion: "0.20.0", - modelId: "openai/gpt-5.4-mini", - }, - }, - type: "session.started", - }, - { session: { id: "session-bounded-state" } }, - ); - - const state = fakeEve.values.get("braintrust.eve.tracing") as { - spanReferences: unknown[]; - stepStarts: unknown[]; - }; - expect(state.spanReferences).toHaveLength(10_000); - expect(state.stepStarts).toHaveLength(10_000); - }); - - it("does not emit a span for session lifecycle metadata alone", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - - await wildcard?.( - { - data: { - runtime: { - agentId: "agent-session-only", - eveVersion: "0.20.0", - modelId: "openai/gpt-5.4-mini", - }, - }, - type: "session.started", - }, - { session: { id: "session-only" } }, - ); - - expect(await backgroundLogger.drain()).toEqual([]); - }); - - it("records a flat Eve turn with session model metadata", async () => { - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-flat-tree" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const expectedModelMetadata = { - model: "braintrust-eve-mock", - provider: "eve-mock", - }; - - await emit({ - data: { - runtime: { - agentId: "agent-id", - agentName: "eve-test-agent", - eveVersion: "0.20.0", - modelId: "eve-mock/braintrust-eve-mock", - }, - }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "session.started", - }); - await emit({ - data: { sequence: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "turn.started", - }); - await emit({ - data: { - message: "Search then read", - sequence: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "message.received", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "actions.requested", - }); - await emit({ - data: { - error: undefined, - result: { - callId: "call-search", - kind: "tool-result", - output: { hits: ["eve.dev/docs"] }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.050Z" }, - type: "action.result", - }); - await emit({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 0, - turnId: "turn-flat-tree", - usage: { - cacheReadTokens: 3, - cacheWriteTokens: 2, - costUsd: 0.001, - inputTokens: 10, - outputTokens: 5, - }, - }, - meta: { at: "2026-01-01T00:00:00.060Z" }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.070Z" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Here is the Eve instrumentation guide.", - sequence: 0, - stepIndex: 1, - turnId: "turn-flat-tree", - }, - meta: { at: "2026-01-01T00:00:00.080Z" }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 1, - turnId: "turn-flat-tree", - usage: { - inputTokens: 20, - outputTokens: 8, - }, - }, - meta: { at: "2026-01-01T00:00:00.090Z" }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-flat-tree" }, - meta: { at: "2026-01-01T00:00:00.100Z" }, - type: "turn.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const root = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - const steps = spans.filter((span) => - String(span.span_attributes?.name).startsWith("eve.step"), - ); - const tool = spans.find((span) => span.span_attributes?.name === "search"); - - expect(spans.map((span) => span.span_attributes?.name)).toEqual([ - "eve.turn", - "eve.step", - "search", - "eve.step", - ]); - expect(root).toMatchObject({ - input: [{ content: "Search then read", role: "user" }], - metadata: { - ...expectedModelMetadata, - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - metrics: { - completion_tokens: 13, - estimated_cost: 0.001, - prompt_cached_tokens: 3, - prompt_cache_creation_tokens: 2, - prompt_tokens: 30, - tokens: 43, - }, - output: "Here is the Eve instrumentation guide.", - span_attributes: { - name: "eve.turn", - type: "task", - }, - root_span_id: deterministicEveIdForTest( - "eve:root", - "session-flat-tree", - "turn-flat-tree", - ), - span_parents: [], - }); - expect(steps).toHaveLength(2); - expect(steps.map((span) => span.span_attributes?.name)).toEqual([ - "eve.step", - "eve.step", - ]); - expect(steps.map((span) => span.span_attributes?.type)).toEqual([ - "llm", - "llm", - ]); - expect(steps.map((span) => span.span_parents)).toEqual([ - [root?.span_id], - [root?.span_id], - ]); - for (const step of steps) { - expect(step.metadata).toEqual({ - ...expectedModelMetadata, - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }); - } - expect(steps[0]?.input).toBeUndefined(); - expect(steps[1]?.input).toBeUndefined(); - expect(tool).toMatchObject({ - input: { query: "Eve instrumentation" }, - metadata: { - "eve.session_id": "session-flat-tree", - scenario: "eve-plugin-unit", - testRunId: "test-run-flat-tree", - }, - output: { hits: ["eve.dev/docs"] }, - span_attributes: { - name: "search", - type: "tool", - }, - span_parents: [root?.span_id], - }); - expect(tool?.metadata).not.toHaveProperty("model"); - expect(tool?.metadata).not.toHaveProperty("provider"); - expect(steps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - tool_calls: [ - { - function: { - arguments: JSON.stringify({ query: "Eve instrumentation" }), - name: "search", - }, - id: "call-search", - type: "function", - }, - ], - }, - }, - ]); - expect(steps[1]?.output).toMatchObject([ - { - finish_reason: "stop", - message: { - content: "Here is the Eve instrumentation guide.", - role: "assistant", - }, - }, - ]); - }); - - it("records each user message as a separate turn in one session", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { session: { id: "session-multi-turn" } }; - for (const [sequence, message] of [ - [0, "First user message"], - [1, "Second user message"], - ] as const) { - const turnId = `turn-${sequence}`; - await wildcard?.( - { data: { sequence, turnId }, type: "turn.started" }, - ctx, - ); - await wildcard?.( - { data: { message, sequence, turnId }, type: "message.received" }, - ctx, - ); - await wildcard?.( - { data: { sequence, turnId }, type: "turn.completed" }, - ctx, - ); - } - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const turns = spans.filter( - (span) => span.span_attributes?.name === "eve.turn", - ); - - expect( - spans.some((span) => span.span_attributes?.name === "eve.session"), - ).toBe(false); - expect(turns).toHaveLength(2); - expect(turns.map((turn) => turn.span_parents)).toEqual([[], []]); - expect(turns.map((turn) => turn.root_span_id)).toEqual([ - deterministicEveIdForTest("eve:root", "session-multi-turn", "turn-0"), - deterministicEveIdForTest("eve:root", "session-multi-turn", "turn-1"), - ]); - expect(turns[0]?.root_span_id).not.toBe(turns[1]?.root_span_id); - expect(turns.map((turn) => turn.input)).toEqual([ - [{ content: "First user message", role: "user" }], - [{ content: "Second user message", role: "user" }], - ]); - }); - - it("merges incremental tool-call batches without reconstructing later LLM inputs", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-incremental-tools" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-incremental-tools" }, - type: "turn.started", - }); - await emit({ - data: { - message: "Search then read", - sequence: 0, - turnId: "turn-incremental-tools", - }, - type: "message.received", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-incremental-tools" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "actions.requested", - }); - await emit({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "Updated Eve instrumentation" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "actions.requested", - }); - await emit({ - data: { - actions: [ - { - callId: "call-read", - input: { url: "https://eve.dev/docs/guides/instrumentation" }, - kind: "tool-call", - toolName: "read", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "actions.requested", - }); - await emit({ - data: { - result: { - callId: "call-search", - kind: "tool-result", - output: { url: "https://eve.dev/docs/guides/instrumentation" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "action.result", - }); - await emit({ - data: { - result: { - callId: "call-read", - kind: "tool-result", - output: { excerpt: "Eve hooks expose runtime stream events." }, - toolName: "read", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "action.result", - }); - await emit({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 0, - turnId: "turn-incremental-tools", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-incremental-tools" }, - type: "step.started", - }); - await emit({ - data: { - finishReason: "stop", - message: "Done.", - sequence: 0, - stepIndex: 1, - turnId: "turn-incremental-tools", - }, - type: "message.completed", - }); - await emit({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 1, - turnId: "turn-incremental-tools", - }, - type: "step.completed", - }); - await emit({ - data: { sequence: 0, turnId: "turn-incremental-tools" }, - type: "turn.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const steps = spans.filter((span) => - String(span.span_attributes?.name).startsWith("eve.step"), - ); - - expect(steps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - tool_calls: [ - { - function: { - arguments: JSON.stringify({ - query: "Updated Eve instrumentation", - }), - name: "search", - }, - id: "call-search", - type: "function", - }, - { - function: { - arguments: JSON.stringify({ - url: "https://eve.dev/docs/guides/instrumentation", - }), - name: "read", - }, - id: "call-read", - type: "function", - }, - ], - }, - }, - ]); - expect(steps[1]?.input).toBeUndefined(); - }); - - it("preserves reasoning across tool, text, and structured outputs", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-reasoning-outputs" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - const turnId = "turn-reasoning-outputs"; - - await emit({ data: { sequence: 0, turnId }, type: "turn.started" }); - - await emit({ - data: { sequence: 0, stepIndex: 0, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "I should inspect the available tools.", - sequence: 0, - stepIndex: 0, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - finishReason: "tool-calls", - message: null, - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "message.completed", - }); - await emit({ - data: { - actions: [ - { - callId: "call-reasoning-search", - input: { query: "Eve reasoning" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "actions.requested", - }); - await emit({ - data: { finishReason: "tool-calls", sequence: 0, stepIndex: 0, turnId }, - type: "step.completed", - }); - - await emit({ - data: { sequence: 0, stepIndex: 1, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "The tool returned the relevant guide.", - sequence: 0, - stepIndex: 1, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - reasoning: "I can now answer concisely.", - sequence: 0, - stepIndex: 1, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - finishReason: "stop", - message: "Eve exposes completed reasoning events.", - sequence: 0, - stepIndex: 1, - turnId, - }, - type: "message.completed", - }); - await emit({ - data: { finishReason: "stop", sequence: 0, stepIndex: 1, turnId }, - type: "step.completed", - }); - - await emit({ - data: { sequence: 0, stepIndex: 2, turnId }, - type: "step.started", - }); - await emit({ - data: { - reasoning: "The response must match the requested schema.", - sequence: 0, - stepIndex: 2, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "reasoning.completed", - }); - await emit({ - data: { - result: { supported: true }, - sequence: 0, - stepIndex: 2, - turnId, - }, - type: "result.completed", - }); - await emit({ - data: { finishReason: "stop", sequence: 0, stepIndex: 2, turnId }, - type: "step.completed", - }); - await emit({ data: { sequence: 0, turnId }, type: "turn.completed" }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const steps = spans.filter( - (span) => span.span_attributes?.name === "eve.step", - ); - expect(steps).toHaveLength(3); - expect(steps[0]?.output).toMatchObject([ - { - finish_reason: "tool_calls", - message: { - content: null, - reasoning: [{ content: "I should inspect the available tools." }], - tool_calls: [ - { - function: { name: "search" }, - id: "call-reasoning-search", - type: "function", - }, - ], - }, - }, - ]); - expect(steps[1]?.output).toMatchObject([ - { - finish_reason: "stop", - message: { - content: "Eve exposes completed reasoning events.", - reasoning: [ - { content: "The tool returned the relevant guide." }, - { content: "I can now answer concisely." }, - ], - }, - }, - ]); - expect(steps[2]?.output).toMatchObject([ - { - finish_reason: "stop", - message: { - content: { supported: true }, - reasoning: [ - { content: "The response must match the requested schema." }, - ], - }, - }, - ]); - }); - - it("rehydrates reasoning idempotently across Vercel Workflow steps", async () => { - const eveState = createFakeDefineState(); - const ctx: EveHookContext = { - session: { id: "session-reasoning-replay" }, - }; - const turnId = "turn-reasoning-replay"; - const reasoningEvent = { - data: { - reasoning: "Use the search tool before answering.", - sequence: 0, - stepIndex: 0, - turnId, - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "reasoning.completed", - } as const satisfies EveHandleMessageStreamEvent; - const initialHook = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - - await initialHook?.( - { - data: { sequence: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }, - ctx, - ); - await initialHook?.( - { - data: { sequence: 0, stepIndex: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }, - ctx, - ); - await initialHook?.(reasoningEvent, ctx); - - const initialWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - expect( - initialWrites - .filter((span) => span.span_attributes?.name) - .every( - (span) => span.context?.span_origin?.instrumentation?.name === "eve", - ), - ).toBe(true); - for (const [key, value] of eveState.values) { - eveState.values.set(key, JSON.parse(JSON.stringify(value))); - } - - const resumedHook = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - await resumedHook?.( - { - data: { sequence: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }, - ctx, - ); - await resumedHook?.( - { - data: { sequence: 0, stepIndex: 0, turnId }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }, - ctx, - ); - const resumedStartWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - await resumedHook?.(reasoningEvent, ctx); - await resumedHook?.( - { - data: { - finishReason: "tool-calls", - message: null, - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "message.completed", - }, - ctx, - ); - await resumedHook?.( - { - data: { - actions: [ - { - callId: "call-replayed-search", - input: { query: "Vercel Workflow reasoning" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId, - }, - type: "actions.requested", - }, - ctx, - ); - const resumedIntermediateWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - await resumedHook?.( - { - data: { finishReason: "tool-calls", sequence: 0, stepIndex: 0, turnId }, - type: "step.completed", - }, - ctx, - ); - await resumedHook?.( - { data: { sequence: 0, turnId }, type: "turn.completed" }, - ctx, - ); - - const resumedWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - const allWrites = [ - ...initialWrites, - ...resumedStartWrites, - ...resumedIntermediateWrites, - ...resumedWrites, - ]; - const stepRowId = deterministicEveIdForTest( - "eve:row:step", - "session-reasoning-replay", - turnId, - "0", - ); - expect( - allWrites.filter( - (write) => write.id === stepRowId && write.output !== undefined, - ), - ).toHaveLength(1); - - const spans = mergeRowBatch([...allWrites].reverse()); - const steps = spans.filter( - (span) => span.span_attributes?.name === "eve.step", - ); - expect(steps).toHaveLength(1); - expect(steps[0]?.span_id).toBe( - deterministicEveIdForTest( - "eve:step", - "session-reasoning-replay", - turnId, - "0", - ), - ); - expect(steps[0]?.output?.[0]?.message).toMatchObject({ - content: null, - reasoning: [{ content: "Use the search tool before answering." }], - tool_calls: [ - { - function: { name: "search" }, - id: "call-replayed-search", - type: "function", - }, - ], - }); - expect(steps[0]?.output?.[0]?.message?.reasoning).toHaveLength(1); - expect(eveState.values.get("braintrust.eve.tracing")).toMatchObject({ - reasoningBlocks: [], - stepStarts: [], - }); - }); - - it("merges late tool results into tool spans closed by turn completion", async () => { - const eveState = createFakeDefineState(); - const wildcard = braintrustEveHook({ - defineState: eveState.defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-late-tool-result" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-late-search", - input: { query: "late tool result" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-late-tool-result", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "actions.requested", - }); - await emit({ - data: { sequence: 0, turnId: "turn-late-tool-result" }, - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "turn.completed", - }); - expect(eveState.values.get("braintrust.eve.tracing")).toMatchObject({ - spanReferences: expect.arrayContaining([ - expect.objectContaining({ - exported: expect.any(String), - rootSpanId: deterministicEveIdForTest( - "eve:root", - "session-late-tool-result", - "turn-late-tool-result", - ), - rowId: deterministicEveIdForTest( - "eve:row:tool", - "session-late-tool-result", - "turn-late-tool-result", - "call-late-search", - ), - spanId: deterministicEveIdForTest( - "eve:tool", - "session-late-tool-result", - "turn-late-tool-result", - "call-late-search", - ), - }), - ]), - stepStarts: [], - }); - const initialWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - const resumedWildcard = braintrustEveHook({ - defineState: eveState.defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - }).events?.["*"]; - const flushSpy = vi.spyOn(backgroundLogger, "flush"); - flushSpy.mockClear(); - await resumedWildcard?.( - { - data: { - result: { - callId: "call-late-search", - kind: "tool-result", - output: { title: "Late result" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-late-tool-result", - }, - meta: { at: "2026-01-01T00:00:00.040Z" }, - type: "action.result", - }, - ctx, - ); - expect(flushSpy).not.toHaveBeenCalled(); - - const resumedWrites = (await backgroundLogger.drain()) as Array< - Record & { id: string } - >; - expect( - [...initialWrites, ...resumedWrites].every( - (span) => span._is_merge === true, - ), - ).toBe(true); - - // The backend may ingest separate workflow uploads out of order. Because - // every write is a merge, a delayed initial write cannot erase the result. - const spans = mergeRowBatch([...resumedWrites, ...initialWrites]); - const turns = spans.filter( - (span) => span.span_attributes?.name === "eve.turn", - ); - const tool = spans.find((span) => span.span_attributes?.name === "search"); - - expect(turns).toHaveLength(1); - expect( - spans.filter((span) => span.span_attributes?.name === "search"), - ).toHaveLength(1); - expect(tool).toMatchObject({ - input: { query: "late tool result" }, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-tool-result", - }, - output: { title: "Late result" }, - span_attributes: { - name: "search", - type: "tool", - }, - span_parents: [turns[0]?.span_id], - }); - expect(tool?.metrics?.end).toEqual(expect.any(Number)); - }); - - it("lets action results complete sparse subagent events across workflow steps", async () => { - const eveState = createFakeDefineState(); - const ctx: EveHookContext = { - session: { id: "session-sparse-subagent" }, - }; - const firstWildcard = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - - await firstWildcard?.( - { - data: { sequence: 0, turnId: "turn-sparse-subagent" }, - type: "turn.started", - }, - ctx, - ); - await firstWildcard?.( - { - data: { - actions: [ - { - callId: "call-sparse-subagent", - input: { message: "Research Eve" }, - kind: "subagent-call", - subagentName: "researcher", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-sparse-subagent", - }, - type: "actions.requested", - }, - ctx, - ); - await firstWildcard?.( - { - data: { - callId: "call-sparse-subagent", - sequence: 0, - subagentName: "researcher", - turnId: "turn-sparse-subagent", - }, - meta: { at: "2026-01-01T00:00:00.100Z" }, - type: "subagent.completed", - }, - ctx, - ); - - const resumedWildcard = braintrustEveHook({ - defineState: eveState.defineState, - }).events?.["*"]; - await resumedWildcard?.( - { - data: { - result: { - callId: "call-sparse-subagent", - kind: "subagent-result", - output: { answer: "Authoritative result" }, - subagentName: "researcher", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-sparse-subagent", - }, - meta: { at: "2026-01-01T00:00:00.500Z" }, - type: "action.result", - }, - ctx, - ); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const subagents = spans.filter( - (span) => span.span_attributes?.name === "researcher", - ); - expect(subagents).toHaveLength(1); - expect(subagents[0]).toMatchObject({ - output: { answer: "Authoritative result" }, - metrics: { end: Date.parse("2026-01-01T00:00:00.100Z") / 1000 }, - span_attributes: { type: "tool" }, - }); - }); - - it("evicts tracing state after session completion", async () => { - const fakeEve = createFakeDefineState(); - defineState = fakeEve.defineState; - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-late-after-session", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-late-after-session" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { sequence: 0, turnId: "turn-late-after-session" }, - meta: { at: "2026-01-01T00:00:00.000Z" }, - type: "turn.started", - }); - await emit({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-late-after-session" }, - meta: { at: "2026-01-01T00:00:00.010Z" }, - type: "step.started", - }); - await emit({ - data: { - actions: [ - { - callId: "call-after-session", - input: { query: "after session" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-late-after-session", - }, - meta: { at: "2026-01-01T00:00:00.020Z" }, - type: "actions.requested", - }); - await emit({ - meta: { at: "2026-01-01T00:00:00.030Z" }, - type: "session.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(tool).toMatchObject({ - input: { query: "after session" }, - }); - expect(tool?.metrics?.end).toEqual(expect.any(Number)); - expect(fakeEve.values.get("braintrust.eve.tracing")).toEqual({ - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }); - }); - - it("records result-only tool events without invented input", async () => { - const wildcard = braintrustEveHook({ - defineState, - metadata: { - scenario: "eve-plugin-unit", - testRunId: "test-run-result-only", - }, - }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-result-only" }, - }; - const emit = (event: EveHandleMessageStreamEvent) => wildcard?.(event, ctx); - - await emit({ - data: { - result: { - callId: "call-result-only", - kind: "tool-result", - output: { title: "Result only" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-result-only", - }, - type: "action.result", - }); - await emit({ - data: { sequence: 0, turnId: "turn-result-only" }, - type: "turn.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const tool = spans.find((span) => span.span_attributes?.name === "search"); - expect(tool).toMatchObject({ - output: { title: "Result only" }, - span_attributes: { name: "search", type: "tool" }, - }); - expect(tool?.input).toBeUndefined(); - }); - - it("evicts tracing state after session failure", async () => { - const fakeEve = createFakeDefineState(); - const wildcard = braintrustEveHook({ - defineState: fakeEve.defineState, - }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-failed-cleanup" }, - }; - - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-failed-cleanup" }, - type: "turn.started", - }, - ctx, - ); - await wildcard?.( - { - data: { - sequence: 0, - stepIndex: 0, - turnId: "turn-failed-cleanup", - }, - type: "step.started", - }, - ctx, - ); - await wildcard?.( - { - data: { - code: "session_failed", - message: "Session failed", - sessionId: "session-failed-cleanup", - }, - type: "session.failed", - }, - ctx, - ); - - expect(fakeEve.values.get("braintrust.eve.tracing")).toEqual({ - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }); - }); - - it("flushes final session events but not ordinary or ignored events", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - const ctx: EveHookContext = { - session: { id: "session-selective-flush" }, - }; - - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-selective-flush" }, - type: "turn.started", - }, - ctx, - ); - const flushSpy = vi - .spyOn(backgroundLogger, "flush") - .mockResolvedValue(undefined); - - await wildcard?.( - { - data: { wait: "next-user-message" }, - type: "session.waiting", - }, - ctx, - ); - expect(flushSpy).not.toHaveBeenCalled(); - - await wildcard?.( - { - data: { - finishReason: "stop", - message: null, - sequence: 0, - stepIndex: 0, - turnId: "turn-selective-flush", - }, - type: "message.completed", - }, - ctx, - ); - expect(flushSpy).not.toHaveBeenCalled(); - - await wildcard?.( - { - type: "session.completed", - }, - ctx, - ); - expect(flushSpy).toHaveBeenCalledTimes(1); - }); - - it("serializes events per session without blocking other sessions", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const sessionA: EveHookContext = { session: { id: "session-queue-a" } }; - const sessionB: EveHookContext = { session: { id: "session-queue-b" } }; - const emitA = (event: EveHandleMessageStreamEvent) => - wildcard?.(event, sessionA); - const emitB = (event: EveHandleMessageStreamEvent) => - wildcard?.(event, sessionB); - - await emitA({ - data: { sequence: 0, turnId: "turn-a" }, - type: "turn.started", - }); - await emitB({ - data: { sequence: 0, turnId: "turn-b" }, - type: "turn.started", - }); - - let releaseFirstFlush: (() => void) | undefined; - const firstFlush = new Promise((resolve) => { - releaseFirstFlush = resolve; - }); - const flushSpy = vi - .spyOn(backgroundLogger, "flush") - .mockImplementationOnce(() => firstFlush) - .mockResolvedValue(undefined); - - const doneA = emitA({ - type: "session.completed", - }); - for (let i = 0; i < 10 && flushSpy.mock.calls.length < 1; i++) { - await new Promise((resolve) => setTimeout(resolve, 0)); - } - expect(flushSpy).toHaveBeenCalledTimes(1); - - let queuedEventFinished = false; - const queuedA = Promise.resolve( - emitA({ - data: { - finishReason: "stop", - message: "queued", - sequence: 0, - stepIndex: 0, - turnId: "turn-a", - }, - type: "message.completed", - }), - ).then(() => { - queuedEventFinished = true; - }); - await Promise.resolve(); - expect(queuedEventFinished).toBe(false); - - await emitB({ - data: { sequence: 0, turnId: "turn-b" }, - type: "turn.completed", - }); - expect(queuedEventFinished).toBe(false); - - releaseFirstFlush?.(); - await Promise.all([doneA, queuedA]); - expect(queuedEventFinished).toBe(true); - }); - - it("uses deterministic ids and attaches local subagent turns to their tool span", async () => { - const parentEveState = createFakeDefineState(); - const childEveState = createFakeDefineState(); - const parentWildcard = braintrustEveHook({ - defineState: parentEveState.defineState, - }).events?.["*"]; - const childWildcard = braintrustEveHook({ - defineState: childEveState.defineState, - }).events?.["*"]; - expect(parentWildcard).toBeDefined(); - expect(childWildcard).toBeDefined(); - - const parentCtx: EveHookContext = { - session: { id: "session-parent" }, - }; - const childCtx: EveHookContext = { - session: { - id: "session-child", - parent: { - callId: "call-researcher", - sessionId: "session-parent", - turn: { id: "turn-parent" }, - }, - }, - }; - const emitParent = (event: EveHandleMessageStreamEvent) => - parentWildcard?.(event, parentCtx); - const emitChild = (event: EveHandleMessageStreamEvent) => - childWildcard?.(event, childCtx); - - await emitParent({ - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.started", - }); - await emitParent({ - data: { - message: "Research Eve tracing", - sequence: 0, - turnId: "turn-parent", - }, - type: "message.received", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - actions: [ - { - callId: "call-researcher", - input: { message: "Find the relevant section" }, - kind: "subagent-call", - name: "researcher", - subagentName: "researcher", - }, - ], - sequence: 0, - stepIndex: 0, - turnId: "turn-parent", - }, - type: "actions.requested", - }); - await emitParent({ - data: { - callId: "call-researcher", - childSessionId: "session-child", - name: "researcher", - sequence: 0, - toolName: "researcher", - turnId: "turn-parent", - }, - type: "subagent.called", - }); - - await emitChild({ - data: { sequence: 0, turnId: "turn-child" }, - type: "turn.started", - }); - await emitChild({ - data: { - message: "Find the relevant section", - sequence: 0, - turnId: "turn-child", - }, - type: "message.received", - }); - await emitChild({ - data: { sequence: 0, stepIndex: 0, turnId: "turn-child" }, - type: "step.started", - }); - await emitChild({ - data: { - actions: [ - { - callId: "call-search", - input: { query: "nested eve" }, - kind: "tool-call", - toolName: "search", - }, - ], - sequence: 0, - stepIndex: 1, - turnId: "turn-child", - }, - type: "actions.requested", - }); - await emitChild({ - data: { - result: { - callId: "call-search", - kind: "tool-result", - output: { title: "Nested Eve" }, - toolName: "search", - }, - sequence: 0, - status: "completed", - stepIndex: 1, - turnId: "turn-child", - }, - type: "action.result", - }); - await emitChild({ - data: { - finishReason: "stop", - message: "Child found Nested Eve.", - sequence: 0, - stepIndex: 0, - turnId: "turn-child", - }, - type: "message.completed", - }); - await emitChild({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 0, - turnId: "turn-child", - }, - type: "step.completed", - }); - await emitChild({ - data: { sequence: 0, turnId: "turn-child" }, - type: "turn.completed", - }); - - await emitParent({ - data: { - callId: "call-researcher", - output: "Child found Nested Eve.", - sequence: 0, - status: "completed", - subagentName: "researcher", - turnId: "turn-parent", - }, - type: "subagent.completed", - }); - await emitParent({ - data: { - result: { - callId: "call-researcher", - kind: "subagent-result", - output: "Child found Nested Eve.", - subagentName: "researcher", - }, - sequence: 0, - status: "completed", - stepIndex: 0, - turnId: "turn-parent", - }, - type: "action.result", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 1, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - actions: [ - { - callId: "call-read", - input: { url: "https://eve.dev/docs/guides/instrumentation" }, - kind: "tool-call", - toolName: "read", - }, - ], - sequence: 0, - stepIndex: 1, - turnId: "turn-parent", - }, - type: "actions.requested", - }); - await emitParent({ - data: { - result: { - callId: "call-read", - kind: "tool-result", - output: { title: "Runtime context" }, - toolName: "read", - }, - sequence: 0, - status: "completed", - stepIndex: 1, - turnId: "turn-parent", - }, - type: "action.result", - }); - await emitParent({ - data: { - finishReason: "tool-calls", - sequence: 0, - stepIndex: 1, - turnId: "turn-parent", - }, - type: "step.completed", - }); - await emitParent({ - data: { sequence: 0, stepIndex: 2, turnId: "turn-parent" }, - type: "step.started", - }); - await emitParent({ - data: { - finishReason: "stop", - message: "Parent used the child result.", - sequence: 0, - stepIndex: 2, - turnId: "turn-parent", - }, - type: "message.completed", - }); - await emitParent({ - data: { - finishReason: "stop", - sequence: 0, - stepIndex: 2, - turnId: "turn-parent", - }, - type: "step.completed", - }); - await emitParent({ - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.completed", - }); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const parentTurnId = deterministicEveIdForTest( - "eve:turn", - "session-parent", - "turn-parent", - ); - const childTurnId = deterministicEveIdForTest( - "eve:turn", - "session-child", - "turn-child", - ); - const subagentSpanId = deterministicEveIdForTest( - "eve:subagent", - "session-parent", - "call-researcher", - ); - const parentTurn = spans.find( - (span) => - span.span_attributes?.name === "eve.turn" && - span.span_id === parentTurnId, - ); - const subagentSpans = spans.filter( - (span) => span.span_attributes?.name === "researcher", - ); - const childTurn = spans.find( - (span) => - span.span_attributes?.name === "eve.turn" && - span.span_id === childTurnId, - ); - const childSearch = spans.find( - (span) => - span.span_attributes?.name === "search" && - span.span_parents?.[0] === childTurnId, - ); - const parentRead = spans.find( - (span) => - span.span_attributes?.name === "read" && - span.span_parents?.[0] === parentTurnId, - ); - const parentSteps = spans.filter( - (span) => - span.span_attributes?.name === "eve.step" && - span.span_parents?.[0] === parentTurnId, - ); - - expect(parentTurn).toBeDefined(); - expect(subagentSpans).toHaveLength(1); - expect(subagentSpans[0]?.span_id).toBe(subagentSpanId); - expect(subagentSpans[0]?.input).toEqual({ - message: "Find the relevant section", - }); - expect(childTurn).toBeDefined(); - expect(childSearch).toBeDefined(); - expect(parentRead).toBeDefined(); - expect(parentSteps).toHaveLength(3); - expect(parentTurn?.span_parents).toEqual([]); - expect(parentTurn?.span_id).toBe(parentTurnId); - expect(parentTurn?.span_id).toMatch( - /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, - ); - expect(parentTurn?.root_span_id).toMatch( - /^([0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/, - ); - expect(parentTurn?.root_span_id).toBe( - deterministicEveIdForTest("eve:root", "session-parent", "turn-parent"), - ); - expect(subagentSpans[0]?.span_parents).toEqual([parentTurn?.span_id]); - expect(childTurn?.span_parents).toEqual([subagentSpanId]); - expect(childTurn?.root_span_id).toBe(parentTurn?.root_span_id); - expect(parentTurn?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(subagentSpans[0]?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(childTurn?.metadata).toEqual({ - "eve.session_id": "session-child", - }); - expect(childSearch?.metadata).toEqual({ - "eve.session_id": "session-child", - }); - expect(parentRead?.metadata).toEqual({ - "eve.session_id": "session-parent", - }); - expect(childSearch?.span_parents).toEqual([childTurn?.span_id]); - expect(childSearch?.span_id).toBe( - deterministicEveIdForTest( - "eve:tool", - "session-child", - "turn-child", - "call-search", - ), - ); - expect(parentRead?.span_id).toBe( - deterministicEveIdForTest( - "eve:tool", - "session-parent", - "turn-parent", - "call-read", - ), - ); - expect(parentSteps.map((span) => span.span_id)).toEqual([ - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "0", - ), - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "1", - ), - deterministicEveIdForTest( - "eve:step", - "session-parent", - "turn-parent", - "2", - ), - ]); - expect(spans.map((span) => span.span_attributes?.name)).toEqual([ - "eve.turn", - "eve.step", - "researcher", - "eve.turn", - "eve.step", - "search", - "eve.step", - "read", - "eve.step", - ]); - - backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); - initLogger({ - projectName: "eve-plugin.test.ts", - projectId: "test-project-id", - }); - const replay = braintrustEveHook({ - defineState: parentEveState.defineState, - }).events?.["*"]; - await replay?.( - { - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.started", - }, - parentCtx, - ); - await replay?.( - { - data: { sequence: 0, turnId: "turn-parent" }, - type: "turn.completed", - }, - parentCtx, - ); - const replaySpans = (await backgroundLogger.drain()) as Array< - Record - >; - expect( - replaySpans.find((span) => span.span_id === parentTurn?.span_id), - ).toMatchObject({ _is_merge: true }); - }); - - it("does not parent an Eve turn under the active Braintrust span", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - const ctx: EveHookContext = { - session: { id: "session-wrapped" }, - }; - const parent = startSpan({ name: "workflow" }); - await withCurrent(parent, async () => { - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-wrapped" }, - type: "turn.started", - }, - ctx, - ); - await wildcard?.( - { - data: { sequence: 0, turnId: "turn-wrapped" }, - type: "turn.completed", - }, - ctx, - ); - }); - parent.end(); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - const turn = spans.find( - (span) => span.span_attributes?.name === "eve.turn", - ); - expect(turn?.span_id).toBe( - deterministicEveIdForTest("eve:turn", "session-wrapped", "turn-wrapped"), - ); - expect(turn?.span_parents).toEqual([]); - expect(turn?.root_span_id).toBe( - deterministicEveIdForTest("eve:root", "session-wrapped", "turn-wrapped"), - ); - expect(turn?.root_span_id).not.toBe(parent.rootSpanId); - }); - - it("does not throw when Eve emits malformed events or failures", async () => { - const wildcard = braintrustEveHook({ defineState }).events?.["*"]; - expect(wildcard).toBeDefined(); - - await expect( - wildcard?.({ bad: true } as never, {} as never), - ).resolves.toBeUndefined(); - await expect( - wildcard?.( - { - data: { - code: "boom", - message: "step failed", - sequence: 0, - stepIndex: 0, - turnId: "turn-missing-session", - }, - type: "step.failed", - }, - {} as never, - ), - ).resolves.toBeUndefined(); - await expect( - wildcard?.( - { - data: { runtime: { modelId: 123 } }, - type: "session.started", - } as never, - { session: { id: "session-malformed-runtime" } }, - ), - ).resolves.toBeUndefined(); - - const spans = (await backgroundLogger.drain()) as Array< - Record - >; - expect(spans).toEqual([]); - }); -}); diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts deleted file mode 100644 index ef18f8913..000000000 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ /dev/null @@ -1,2310 +0,0 @@ -import { debugLogger } from "../../debug-logger"; -import { - NOOP_SPAN, - _internalStartSpanWithInitialMerge, - flush, - updateSpan, - withCurrent, -} from "../../logger"; -import type { Span } from "../../logger"; -import { LRUCache } from "../../lru-cache"; -import { - INSTRUMENTATION_NAMES, - withSpanInstrumentationName, -} from "../../span-origin"; -import { SpanTypeAttribute, isObject } from "../../../util/index"; -import { getCurrentUnixTimestamp } from "../../util"; -import type { - EveAssistantStepFinishReason, - EveActionResultError, - EveHandleMessageStreamEvent, - EveHookContext, - EveHookDefinition, - EveInstrumentationDefinition, - EveInstrumentationModelInput, - EveInstrumentationStepStartedEventInput, - EveModelMessage, - EveModelMessageContentPart, - EveRuntimeActionRequest, - EveRuntimeActionResult, - EveRuntimeToolCallActionRequest, - EveRuntimeToolResultActionResult, -} from "../../vendor-sdk-types/eve"; - -type SpanState = { - metadata: Record; - span: EveSpan; -}; - -type EveSpan = Pick; - -type EveSpanStartEvent = { - readonly created: string; - readonly metrics: { readonly start: number }; - readonly span_attributes: Record; - readonly span_parents: string[]; -}; - -type EveSpanReference = { - readonly endTime?: number; - readonly exported: string; - readonly rootSpanId: string; - readonly rowId: string; - readonly spanId: string; - readonly startEvent?: EveSpanStartEvent; -}; - -type StepState = SpanState & { - input?: unknown; - metrics: Record; - output?: unknown; - reasoning: readonly EveReasoningBlock[]; -}; - -type TurnState = SpanState & { - key: string; - metrics: Record; - output?: unknown; - sessionId: string; - stepsByIndex: Map; - turnId: string; -}; - -type ToolState = SpanState & { - endedByTurn?: boolean; - turnKey: string; -}; - -type EveEntityKind = "step" | "subagent" | "tool" | "turn"; - -type EveStateHandle = { - get(): T; - update(fn: (current: T) => T): void; -}; - -type EveDefineState = (name: string, initial: () => T) => EveStateHandle; - -type EveTraceState = { - metadata: Record; - reasoningBlocks: readonly (EveReasoningBlock & { key: string })[]; - spanReferences: readonly EveSpanReference[]; - stepStarts: readonly { - ordinal: number; - open: boolean; - stepIndex: number; - turnId: string; - }[]; - llmInputs: readonly { - input: CapturedEveModelInput; - key: string; - }[]; -}; - -type EveReasoningBlock = { - content: string; - eventAt?: string; -}; - -const EVE_TRACE_STATE_KEY = "braintrust.eve.tracing"; -const MAX_EVE_CACHE_ENTRIES = 10_000; -const MAX_STORED_LLM_INPUTS = 100; -const MAX_STORED_REASONING_BLOCKS = 100; -const MAX_STORED_SPAN_REFERENCES = 10_000; -const MAX_STORED_STEP_STARTS = 10_000; - -type CapturedEveModelMessage = { - content: string | readonly Record[]; - role: EveModelMessage["role"]; -}; - -type CapturedEveModelInput = readonly CapturedEveModelMessage[]; - -/** Manual hook instrumentation for eve runtime stream events. */ -export function braintrustEveHook(options: { - defineState: EveDefineState; - metadata?: Record; -}): EveHookDefinition { - const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); - const bridge = new EveBridge(state); - return { - events: { - "*": async (event: EveHandleMessageStreamEvent, ctx: EveHookContext) => { - await bridge.handle(event, ctx, options.metadata); - }, - }, - }; -} - -/** Eve instrumentation helper for logger setup and durable LLM input capture. */ -export function braintrustEveInstrumentation(options: { - defineState: EveDefineState; - setup?: EveInstrumentationDefinition["setup"]; -}): EveInstrumentationDefinition { - const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); - return { - events: { - "step.started": (input: EveInstrumentationStepStartedEventInput) => { - try { - captureEveModelInput(state, input); - } catch (error) { - debugLogger.warn("Error in Eve LLM input capture:", error); - } - }, - }, - recordInputs: false, - recordOutputs: false, - setup: options.setup, - }; -} - -function isEveHandleMessageStreamEvent( - event: unknown, -): event is EveHandleMessageStreamEvent { - return isObject(event) && typeof event["type"] === "string"; -} - -class ResumedEveSpan implements EveSpan { - private endTime: number | undefined; - - constructor(private readonly reference: EveSpanReference) { - this.endTime = reference.endTime; - } - - get rootSpanId(): string { - return this.reference.rootSpanId; - } - - get spanId(): string { - return this.reference.spanId; - } - - log(event: Parameters[0]): void { - const metrics = { - ...this.reference.startEvent?.metrics, - ...(this.endTime === undefined ? {} : { end: this.endTime }), - ...event.metrics, - }; - updateSpan({ - exported: this.reference.exported, - ...this.reference.startEvent, - ...event, - ...(Object.keys(metrics).length > 0 ? { metrics } : {}), - }); - } - - end(args?: Parameters[0]): number { - if (this.endTime === undefined) { - this.endTime = args?.endTime ?? getCurrentUnixTimestamp(); - this.log({ metrics: { end: this.endTime } }); - } - return this.endTime; - } -} - -class EveBridge { - constructor(private readonly state: EveStateHandle) {} - - private eventQueuesBySession = new Map>(); - private completedToolKeys = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); - private toolsByCallKey = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); - private turnsByKey = new LRUCache({ - max: MAX_EVE_CACHE_ENTRIES, - }); - - private async startEveSpan( - args: Parameters[0], - ): Promise { - const rowId = args?.event?.id; - const reference = - typeof rowId === "string" && - readEveTraceState(this.state).spanReferences.find( - (candidate) => candidate.rowId === rowId, - ); - if (reference) { - return new ResumedEveSpan(reference); - } - - const startTime = args?.startTime ?? getCurrentUnixTimestamp(); - const parentSpanIds = args?.parentSpanIds; - const startEvent: EveSpanStartEvent = { - created: new Date().toISOString(), - metrics: { start: startTime }, - span_attributes: { - ...(args?.name ? { name: args.name } : {}), - ...(args?.type ? { type: args.type } : {}), - ...args?.spanAttributes, - }, - span_parents: parentSpanIds - ? "spanId" in parentSpanIds - ? [parentSpanIds.spanId] - : parentSpanIds.parentSpanIds - : [], - }; - const span = withCurrent(NOOP_SPAN, () => - _internalStartSpanWithInitialMerge( - withSpanInstrumentationName( - { ...args, startTime }, - INSTRUMENTATION_NAMES.EVE, - ), - ), - ); - if (typeof rowId !== "string") { - return span; - } - - try { - const exported = await span.export(); - const reference = { - exported, - rootSpanId: span.rootSpanId, - rowId, - spanId: span.spanId, - startEvent, - }; - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return normalized.spanReferences.some( - (candidate) => candidate.rowId === rowId, - ) - ? normalized - : { - ...normalized, - spanReferences: [...normalized.spanReferences, reference].slice( - -MAX_STORED_SPAN_REFERENCES, - ), - }; - }); - } catch (error) { - debugLogger.warn("Error exporting Eve span for resumption:", error); - } - return span; - } - - private async startEveChildSpan( - parent: EveSpan, - args: Parameters[0], - ): Promise { - return await this.startEveSpan({ - ...args, - parentSpanIds: { - rootSpanId: parent.rootSpanId, - spanId: parent.spanId, - }, - }); - } - - private stepOrdinal( - event: Extract, - ): number { - let ordinal = 0; - this.state.update((current) => { - const state = normalizeEveTraceState(current); - const previous = state.stepStarts - .filter( - (entry) => - entry.turnId === event.data.turnId && - entry.stepIndex === event.data.stepIndex, - ) - .at(-1); - if (previous?.open) { - ordinal = previous.ordinal; - return state; - } - - ordinal = state.stepStarts.filter( - (entry) => entry.turnId === event.data.turnId, - ).length; - return { - ...state, - stepStarts: [ - ...state.stepStarts, - { - open: true, - ordinal, - stepIndex: event.data.stepIndex, - turnId: event.data.turnId, - }, - ].slice(-MAX_STORED_STEP_STARTS), - }; - }); - return ordinal; - } - - private markStepEnded(turnId: string, stepIndex: number): void { - this.state.update((current) => { - const state = normalizeEveTraceState(current); - let index = -1; - for (let i = state.stepStarts.length - 1; i >= 0; i--) { - const entry = state.stepStarts[i]; - if (entry?.turnId === turnId && entry.stepIndex === stepIndex) { - index = i; - break; - } - } - if (index < 0 || !state.stepStarts[index]?.open) { - return state; - } - return { - ...state, - stepStarts: state.stepStarts.map((entry, entryIndex) => - entryIndex === index ? { ...entry, open: false } : entry, - ), - }; - }); - } - - async handle( - event: unknown, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (!isEveHandleMessageStreamEvent(event)) { - return; - } - const run = async () => { - try { - if (!(await this.handleEvent(event, ctx, hookMetadata))) { - return; - } - if (event.type === "session.failed") { - const sessionId = event.data.sessionId || sessionIdFromContext(ctx); - await this.flushInstrumentation(); - if (sessionId) { - this.cleanupSession(sessionId); - } - } else if (event.type === "session.completed") { - const sessionId = sessionIdFromContext(ctx); - await this.flushInstrumentation(); - if (sessionId) { - this.cleanupSession(sessionId); - } - } - } catch (error) { - debugLogger.warn("Error in Eve hook instrumentation:", error); - } - }; - - const sessionId = - event.type === "session.failed" - ? event.data.sessionId || sessionIdFromContext(ctx) - : sessionIdFromContext(ctx); - if (!sessionId) { - await run(); - return; - } - - const previous = this.eventQueuesBySession.get(sessionId); - const queued = previous ? previous.then(run) : run(); - this.eventQueuesBySession.set(sessionId, queued); - try { - await queued; - } finally { - if (this.eventQueuesBySession.get(sessionId) === queued) { - this.eventQueuesBySession.delete(sessionId); - } - } - } - - private async handleEvent( - event: EveHandleMessageStreamEvent, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - switch (event.type) { - case "session.started": - this.handleSessionStarted(event, ctx, hookMetadata); - return true; - case "turn.started": - await this.handleTurnStarted(event, ctx, hookMetadata); - return true; - case "message.received": - await this.handleMessageReceived(event, ctx, hookMetadata); - return true; - case "step.started": - await this.handleStepStarted(event, ctx, hookMetadata); - return true; - case "reasoning.completed": - this.handleReasoningCompleted(event, ctx); - return true; - case "message.completed": - this.handleMessageCompleted(event, ctx); - return true; - case "result.completed": - this.handleResultCompleted(event, ctx); - return true; - case "actions.requested": - await this.handleActionsRequested(event, ctx, hookMetadata); - return true; - case "action.result": - await this.handleActionResult(event, ctx, hookMetadata); - return true; - case "subagent.called": - await this.handleSubagentCalled(event, ctx, hookMetadata); - return true; - case "subagent.completed": - await this.handleSubagentCompleted(event, ctx, hookMetadata); - return true; - case "step.completed": - this.handleStepCompleted(event, ctx); - return true; - case "step.failed": - this.handleStepFailed(event, ctx); - return true; - case "turn.completed": - this.handleTurnCompleted(event, ctx); - return true; - case "turn.failed": - this.handleTurnFailed(event, ctx); - return true; - case "session.failed": - this.handleSessionFailed(event, ctx); - return true; - case "session.completed": - this.handleSessionCompleted(event, ctx); - return true; - default: - return false; - } - } - - private handleSessionStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const metadata = { - ...(hookMetadata ?? {}), - ...modelMetadataFromRuntime(event.data.runtime), - }; - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - metadata: { ...normalized.metadata, ...metadata }, - }; - }); - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - - turn.metadata = { ...turn.metadata, ...metadata }; - turn.span.log({ metadata: turn.metadata }); - for (const step of turn.stepsByIndex.values()) { - step.metadata = { ...step.metadata, ...metadata }; - step.span.log({ metadata: step.metadata }); - } - } - } - - private async handleTurnStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = turnKey(sessionId, event.data.turnId); - const metadata = { - ...readEveTraceState(this.state).metadata, - ...(hookMetadata ?? {}), - "eve.session_id": sessionId, - }; - const existing = this.turnsByKey.get(key); - if (existing) { - existing.metadata = { ...existing.metadata, ...metadata }; - existing.span.log({ metadata: existing.metadata }); - return; - } - - const span = await this.startTurnSpan(sessionId, event, ctx, metadata); - span.log({ metadata }); - this.turnsByKey.set(key, { - key, - metadata, - metrics: {}, - sessionId, - span, - stepsByIndex: new Map(), - turnId: event.data.turnId, - }); - } - - private async handleMessageReceived( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - if (!turn) { - return; - } - - const input = [{ content: event.data.message, role: "user" }]; - turn.span.log({ input }); - } - - private async handleStepStarted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; - } - - const existing = turn.stepsByIndex.get(event.data.stepIndex); - if (existing) { - existing.span.log({ - ...(existing.input !== undefined ? { input: existing.input } : {}), - metadata: existing.metadata, - metrics: existing.metrics, - output: existing.output, - }); - const endTime = eventTime(event); - existing.span.end(endTime === undefined ? undefined : { endTime }); - this.markStepEnded(event.data.turnId, event.data.stepIndex); - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } - - const stepOrdinal = this.stepOrdinal(event); - const metadata = { ...turn.metadata }; - const input = consumeCapturedEveModelInput( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - const reasoning = readStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - const output = mergeEveReasoning(undefined, reasoning); - const { rowId: eventId, spanId } = await generateEveIds( - "step", - sessionId, - event.data.turnId, - String(stepOrdinal), - ); - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - ...(input !== undefined ? { input } : {}), - metadata, - }, - name: "eve.step", - spanAttributes: { type: SpanTypeAttribute.LLM }, - spanId, - startTime: eventTime(event), - }); - span.log({ - ...(input !== undefined ? { input } : {}), - metadata, - }); - - turn.stepsByIndex.set(event.data.stepIndex, { - ...(input !== undefined ? { input } : {}), - metadata, - metrics: {}, - ...(output !== undefined ? { output } : {}), - reasoning, - span, - }); - } - - private handleReasoningCompleted( - event: Extract< - EveHandleMessageStreamEvent, - { type: "reasoning.completed" } - >, - ctx: unknown, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const reasoning = storeEveReasoning(this.state, sessionId, event); - const step = this.stepForEvent(event, ctx); - if (step) { - step.reasoning = reasoning; - step.output = mergeEveReasoning(step.output, reasoning); - } - } - - private handleMessageCompleted( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (!step) { - return; - } - - const existingMessage = eveOutputMessage(step.output); - const existingToolCalls = isObject(existingMessage) - ? existingMessage.tool_calls - : undefined; - step.output = mergeEveReasoning( - [ - { - finish_reason: normalizedFinishReason(event.data.finishReason), - index: 0, - message: { - content: event.data.message, - role: "assistant", - ...(Array.isArray(existingToolCalls) - ? { tool_calls: existingToolCalls } - : {}), - }, - }, - ], - step.reasoning, - ); - - const turn = this.turnForEvent(event, ctx); - if (turn && event.data.finishReason !== "tool-calls") { - turn.output = event.data.message; - } - } - - private handleResultCompleted( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (step) { - step.output = mergeEveReasoning( - [ - { - finish_reason: "stop", - index: 0, - message: { - content: event.data.result, - role: "assistant", - }, - }, - ], - step.reasoning, - ); - } - - const turn = this.turnForEvent(event, ctx); - if (turn) { - turn.output = event.data.result; - } - } - - private async handleActionsRequested( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; - } - - const traceActions = event.data.actions.filter(isTraceableActionRequest); - if (traceActions.length === 0) { - return; - } - - for (const action of traceActions) { - if (isToolCallAction(action)) { - await this.startRequestedTool(event, turn, sessionId, action); - } else if (isLocalSubagentCallAction(action)) { - await this.startRequestedSubagent(event, turn, sessionId, action); - } - } - - const step = turn.stepsByIndex.get(event.data.stepIndex); - if (!step) { - return; - } - - const toolCallsById = new Map(); - if (Array.isArray(step.output) && isObject(step.output[0])) { - const message = step.output[0]["message"]; - if (isObject(message) && Array.isArray(message["tool_calls"])) { - for (const toolCall of message["tool_calls"]) { - if (isObject(toolCall) && typeof toolCall["id"] === "string") { - toolCallsById.set(toolCall["id"], toolCall); - } - } - } - } - for (const action of traceActions) { - const name = - action.kind === "tool-call" - ? action.toolName - : (action.subagentName ?? action.name ?? "agent"); - toolCallsById.set(action.callId, { - function: { - arguments: JSON.stringify(action.input), - name, - }, - id: action.callId, - type: "function", - }); - } - - step.output = mergeEveReasoning( - [ - { - finish_reason: "tool_calls", - index: 0, - message: { - content: null, - role: "assistant", - tool_calls: [...toolCallsById.values()], - }, - }, - ], - step.reasoning, - ); - } - - private async handleActionResult( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (isToolResult(event.data.result)) { - await this.handleToolResult(event, ctx, event.data.result, hookMetadata); - return; - } - if (isSubagentResult(event.data.result)) { - await this.handleSubagentResult( - event, - ctx, - event.data.result, - hookMetadata, - ); - } - } - - private async handleToolResult( - event: Extract, - ctx: unknown, - result: EveRuntimeToolResultActionResult, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, result.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const tool = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticTool(event, ctx, result, hookMetadata)); - if (!tool) { - return; - } - const failed = - event.data.status === "failed" || - result.isError === true || - event.data.error !== undefined; - tool.span.log({ - ...(failed - ? { - error: actionResultError(event.data.error, result.output), - } - : {}), - metadata: tool.metadata, - output: result.output, - }); - - const endTime = eventTime(event); - tool.span.end(endTime === undefined ? undefined : { endTime }); - this.toolsByCallKey.delete(key); - this.completedToolKeys.set(key, true); - } - - private async handleSubagentCalled( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - if (event.data.remote?.url) { - return; - } - - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return; - } - - const key = toolKey(sessionId, event.data.callId); - const metadata = toolMetadataFromTurn(turn); - const existing = this.toolsByCallKey.get(key); - if (existing) { - existing.metadata = { ...existing.metadata, ...metadata }; - existing.span.log({ metadata: existing.metadata }); - return; - } - if (this.completedToolKeys.has(key)) { - return; - } - - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - event.data.callId, - ); - const pending = this.toolsByCallKey.get(key); - if (pending || this.completedToolKeys.has(key)) { - if (pending) { - pending.metadata = { ...pending.metadata, ...metadata }; - pending.span.log({ metadata: pending.metadata }); - } - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: event.data.toolName ?? event.data.name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); - } - - private async handleSubagentCompleted( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, event.data.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const subagent = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticSubagent(event, ctx, hookMetadata)); - if (!subagent) { - return; - } - subagent.span.log({ - ...(event.data.status === "failed" - ? { - error: actionResultError(event.data.error, event.data.output), - } - : {}), - metadata: subagent.metadata, - ...(event.data.output !== undefined ? { output: event.data.output } : {}), - }); - const endTime = eventTime(event); - const recordedEndTime = subagent.span.end( - endTime === undefined ? undefined : { endTime }, - ); - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - spanReferences: normalized.spanReferences.map((reference) => - reference.spanId === subagent.span.spanId - ? { ...reference, endTime: recordedEndTime } - : reference, - ), - }; - }); - } - - private async handleSubagentResult( - event: Extract, - ctx: unknown, - result: Extract, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - - const key = toolKey(sessionId, result.callId); - if (this.completedToolKeys.has(key)) { - return; - } - const subagent = - this.toolsByCallKey.get(key) ?? - (await this.startSyntheticSubagentResult( - event, - ctx, - result, - hookMetadata, - )); - if (!subagent) { - return; - } - const isError = - event.data.status === "failed" || - result.isError === true || - event.data.error !== undefined; - subagent.span.log({ - ...(isError - ? { - error: actionResultError(event.data.error, result.output), - } - : {}), - metadata: subagent.metadata, - output: result.output, - }); - const endTime = eventTime(event); - subagent.span.end(endTime === undefined ? undefined : { endTime }); - - this.toolsByCallKey.delete(key); - this.completedToolKeys.set(key, true); - } - - private handleStepCompleted( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (!step) { - return; - } - - const usage = event.data.usage; - const inputTokens = - typeof usage?.inputTokens === "number" && - Number.isFinite(usage.inputTokens) && - usage.inputTokens >= 0 - ? usage.inputTokens - : undefined; - const outputTokens = - typeof usage?.outputTokens === "number" && - Number.isFinite(usage.outputTokens) && - usage.outputTokens >= 0 - ? usage.outputTokens - : undefined; - const cacheReadTokens = - typeof usage?.cacheReadTokens === "number" && - Number.isFinite(usage.cacheReadTokens) && - usage.cacheReadTokens >= 0 - ? usage.cacheReadTokens - : undefined; - const cacheWriteTokens = - typeof usage?.cacheWriteTokens === "number" && - Number.isFinite(usage.cacheWriteTokens) && - usage.cacheWriteTokens >= 0 - ? usage.cacheWriteTokens - : undefined; - const costUsd = - typeof usage?.costUsd === "number" && - Number.isFinite(usage.costUsd) && - usage.costUsd >= 0 - ? usage.costUsd - : undefined; - const total = - inputTokens !== undefined && outputTokens !== undefined - ? inputTokens + outputTokens - : undefined; - const metrics = { - ...(inputTokens !== undefined ? { prompt_tokens: inputTokens } : {}), - ...(outputTokens !== undefined - ? { completion_tokens: outputTokens } - : {}), - ...(total !== undefined ? { tokens: total } : {}), - ...(cacheReadTokens !== undefined - ? { prompt_cached_tokens: cacheReadTokens } - : {}), - ...(cacheWriteTokens !== undefined - ? { prompt_cache_creation_tokens: cacheWriteTokens } - : {}), - ...(costUsd !== undefined ? { estimated_cost: costUsd } : {}), - }; - step.metrics = { ...step.metrics, ...metrics }; - const sessionId = sessionIdFromContext(ctx); - if (Array.isArray(step.output) && isObject(step.output[0])) { - const finishReason = step.output[0].finish_reason; - if (typeof finishReason !== "string") { - step.output[0].finish_reason = normalizedFinishReason( - event.data.finishReason, - ); - } - } - step.span.log({ - ...(step.input !== undefined ? { input: step.input } : {}), - metadata: step.metadata, - metrics, - output: step.output, - }); - const endTime = eventTime(event); - step.span.end(endTime === undefined ? undefined : { endTime }); - - const turn = this.turnForEvent(event, ctx); - if (turn) { - for (const [key, value] of Object.entries(metrics)) { - turn.metrics[key] = (turn.metrics[key] ?? 0) + value; - } - turn.stepsByIndex.delete(event.data.stepIndex); - } - this.markStepEnded(event.data.turnId, event.data.stepIndex); - if (sessionId) { - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } - } - - private handleStepFailed( - event: Extract, - ctx: unknown, - ): void { - const step = this.stepForEvent(event, ctx); - if (step) { - step.span.log({ - error: errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ), - }); - const endTime = eventTime(event); - step.span.end(endTime === undefined ? undefined : { endTime }); - } - - const turn = this.turnForEvent(event, ctx); - turn?.stepsByIndex.delete(event.data.stepIndex); - this.markStepEnded(event.data.turnId, event.data.stepIndex); - const sessionId = sessionIdFromContext(ctx); - if (sessionId) { - clearStoredEveReasoning( - this.state, - sessionId, - event.data.turnId, - event.data.stepIndex, - ); - } - } - - private handleTurnCompleted( - event: Extract, - ctx: unknown, - ): void { - const turn = this.turnForEvent(event, ctx); - if (!turn) { - return; - } - - this.finalizeTurn(turn, { - endTime: eventTime(event), - }); - } - - private handleTurnFailed( - event: Extract, - ctx: unknown, - ): void { - const turn = this.turnForEvent(event, ctx); - if (!turn) { - return; - } - - this.finalizeTurn(turn, { - endTime: eventTime(event), - error: errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ), - }); - } - - private handleSessionFailed( - event: Extract, - ctx: unknown, - ): void { - const sessionId = event.data.sessionId || sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - const error = errorFromMessage( - event.data.message, - event.data.code, - event.data.details, - ); - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - this.finalizeTurn(turn, { - endTime: eventTime(event), - error, - }); - } - - for (const [key, tool] of this.toolsByCallKey) { - if (key.startsWith(`${sessionId}:`)) { - const endTime = eventTime(event); - if (!tool.endedByTurn) { - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; - } - } - } - } - - private handleSessionCompleted( - event: Extract, - ctx: unknown, - ): void { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return; - } - for (const [key, turn] of this.turnsByKey) { - if (!key.startsWith(`${sessionId}:`)) { - continue; - } - this.finalizeTurn(turn, { - endTime: eventTime(event), - }); - } - - for (const [key, tool] of this.toolsByCallKey) { - if (key.startsWith(`${sessionId}:`) && !tool.endedByTurn) { - const endTime = eventTime(event); - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; - } - } - } - - private async ensureTurn( - event: Extract< - EveHandleMessageStreamEvent, - { - data: { readonly sequence: number; readonly turnId: string }; - } - >, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const sessionId = sessionIdFromContext(ctx); - if (!sessionId) { - return undefined; - } - - const key = turnKey(sessionId, event.data.turnId); - const existing = this.turnsByKey.get(key); - if (existing) { - return existing; - } - - const metadata = { - ...readEveTraceState(this.state).metadata, - ...(hookMetadata ?? {}), - "eve.session_id": sessionId, - }; - const span = await this.startTurnSpan(sessionId, event, ctx, metadata); - span.log({ metadata }); - const state = { - key, - metadata, - metrics: {}, - sessionId, - span, - stepsByIndex: new Map(), - turnId: event.data.turnId, - }; - this.turnsByKey.set(key, state); - return state; - } - - private async startRequestedTool( - event: Extract, - turn: TurnState, - sessionId: string, - action: EveRuntimeToolCallActionRequest, - ): Promise { - const key = toolKey(sessionId, action.callId); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "tool", - sessionId, - event.data.turnId, - action.callId, - ); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - input: action.input, - metadata, - }, - name: action.toolName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ input: action.input, metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); - } - - private async startRequestedSubagent( - event: Extract, - turn: TurnState, - sessionId: string, - action: Extract, - ): Promise { - const key = toolKey(sessionId, action.callId); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - - const name = action.subagentName ?? action.name ?? "agent"; - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - action.callId, - ); - if (this.toolsByCallKey.has(key) || this.completedToolKeys.has(key)) { - return; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - input: action.input, - metadata, - }, - name, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ input: action.input, metadata }); - this.toolsByCallKey.set(key, { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }); - } - - private async startSyntheticTool( - event: Extract, - ctx: unknown, - result: EveRuntimeToolResultActionResult, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "tool", - sessionId, - event.data.turnId, - result.callId, - ); - const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId)); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: result.toolName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, result.callId), state); - return state; - } - - private async startSyntheticSubagent( - event: Extract, - ctx: unknown, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - event.data.callId, - ); - const existing = this.toolsByCallKey.get( - toolKey(sessionId, event.data.callId), - ); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: event.data.subagentName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, event.data.callId), state); - return state; - } - - private async startSyntheticSubagentResult( - event: Extract, - ctx: unknown, - result: Extract, - hookMetadata?: Record, - ): Promise { - const turn = await this.ensureTurn(event, ctx, hookMetadata); - const sessionId = sessionIdFromContext(ctx); - if (!turn || !sessionId) { - return undefined; - } - - const metadata = toolMetadataFromTurn(turn); - const { rowId: eventId, spanId } = await generateEveIds( - "subagent", - sessionId, - result.callId, - ); - const existing = this.toolsByCallKey.get(toolKey(sessionId, result.callId)); - if (existing) { - return existing; - } - const span = await this.startEveChildSpan(turn.span, { - event: { - id: eventId, - metadata, - }, - name: result.subagentName, - spanAttributes: { type: SpanTypeAttribute.TOOL }, - spanId, - startTime: eventTime(event), - }); - span.log({ metadata }); - const state = { - metadata, - span, - turnKey: turnKey(sessionId, event.data.turnId), - }; - this.toolsByCallKey.set(toolKey(sessionId, result.callId), state); - return state; - } - - private async startTurnSpan( - sessionId: string, - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly sequence: number; readonly turnId: string } } - >, - ctx: unknown, - metadata: Record, - ): Promise { - const session = isObject(ctx) ? ctx["session"] : undefined; - const parent = isObject(session) ? session["parent"] : undefined; - const parentTurn = isObject(parent) ? parent["turn"] : undefined; - const parentLineage = - isObject(parent) && - typeof parent["callId"] === "string" && - typeof parent["sessionId"] === "string" && - isObject(parentTurn) && - typeof parentTurn["id"] === "string" - ? { - callId: parent["callId"], - sessionId: parent["sessionId"], - turnId: parentTurn["id"], - } - : undefined; - const [{ rowId: eventId, spanId }, rootSpanId, parentSpanId] = - await Promise.all([ - generateEveIds("turn", sessionId, event.data.turnId), - deterministicEveId( - "eve:root", - parentLineage?.sessionId ?? sessionId, - parentLineage?.turnId ?? event.data.turnId, - ), - parentLineage - ? deterministicEveId( - "eve:subagent", - parentLineage.sessionId, - parentLineage.callId, - ) - : Promise.resolve(undefined), - ]); - - return await this.startEveSpan({ - event: { - id: eventId, - metadata, - }, - name: "eve.turn", - parentSpanIds: parentSpanId - ? { rootSpanId, spanId: parentSpanId } - : { parentSpanIds: [], rootSpanId }, - spanAttributes: { type: SpanTypeAttribute.TASK }, - spanId, - startTime: eventTime(event), - }); - } - - private turnForEvent( - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly turnId: string } } - >, - ctx: unknown, - ): TurnState | undefined { - const sessionId = sessionIdFromContext(ctx); - return sessionId - ? this.turnsByKey.get(turnKey(sessionId, event.data.turnId)) - : undefined; - } - - private stepForEvent( - event: Extract< - EveHandleMessageStreamEvent, - { data: { readonly stepIndex: number; readonly turnId: string } } - >, - ctx: unknown, - ): StepState | undefined { - return this.turnForEvent(event, ctx)?.stepsByIndex.get( - event.data.stepIndex, - ); - } - - private finalizeTurn( - turn: TurnState, - args: { endTime: number | undefined; error?: Error }, - ): void { - const { endTime } = args; - for (const step of turn.stepsByIndex.values()) { - step.span.log({ - ...(step.input !== undefined ? { input: step.input } : {}), - metadata: step.metadata, - metrics: step.metrics, - output: step.output, - }); - step.span.end(endTime === undefined ? undefined : { endTime }); - } - turn.stepsByIndex.clear(); - - for (const tool of this.toolsByCallKey.values()) { - if (tool.turnKey !== turn.key) { - continue; - } - if (tool.endedByTurn) { - continue; - } - tool.span.log({ metadata: tool.metadata }); - tool.span.end(endTime === undefined ? undefined : { endTime }); - tool.endedByTurn = true; - } - - if (args.error) { - turn.span.log({ error: args.error }); - } else { - turn.span.log({ - metadata: turn.metadata, - metrics: turn.metrics, - output: turn.output, - }); - } - turn.span.end(endTime === undefined ? undefined : { endTime }); - this.turnsByKey.delete(turn.key); - this.state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - reasoningBlocks: normalized.reasoningBlocks.filter( - (entry) => - !entry.key.startsWith(`${turn.sessionId}\0${turn.turnId}\0`), - ), - stepStarts: normalized.stepStarts.filter( - (entry) => entry.turnId !== turn.turnId, - ), - }; - }); - } - - private cleanupSession(sessionId: string): void { - const keyPrefix = `${sessionId}:`; - for (const key of this.turnsByKey.keys()) { - if (key.startsWith(keyPrefix)) { - this.turnsByKey.delete(key); - } - } - for (const key of this.toolsByCallKey.keys()) { - if (key.startsWith(keyPrefix)) { - this.toolsByCallKey.delete(key); - } - } - for (const key of this.completedToolKeys.keys()) { - if (key.startsWith(keyPrefix)) { - this.completedToolKeys.delete(key); - } - } - this.state.update(() => emptyEveTraceState()); - } - - private async flushInstrumentation(): Promise { - try { - await flush(); - return true; - } catch (error) { - debugLogger.warn("Error in Eve flush instrumentation:", error); - return false; - } - } -} - -function emptyEveTraceState(): EveTraceState { - return { - llmInputs: [], - metadata: {}, - reasoningBlocks: [], - spanReferences: [], - stepStarts: [], - }; -} - -function normalizeEveTraceState(state: unknown): EveTraceState { - if (!isObject(state)) { - return emptyEveTraceState(); - } - const metadata = isObject(state["metadata"]) ? state["metadata"] : {}; - const spanReferences = Array.isArray(state["spanReferences"]) - ? state["spanReferences"] - .flatMap((entry): EveTraceState["spanReferences"] => { - if (!isObject(entry)) { - return []; - } - const exported = entry["exported"]; - const endTime = entry["endTime"]; - const rootSpanId = entry["rootSpanId"]; - const rowId = entry["rowId"]; - const spanId = entry["spanId"]; - const startEvent = entry["startEvent"]; - const startEventCreated = isObject(startEvent) - ? startEvent["created"] - : undefined; - const startEventMetrics = isObject(startEvent) - ? startEvent["metrics"] - : undefined; - const startEventSpanAttributes = isObject(startEvent) - ? startEvent["span_attributes"] - : undefined; - const startEventSpanParents = isObject(startEvent) - ? startEvent["span_parents"] - : undefined; - const normalizedStartEvent = - typeof startEventCreated === "string" && - isObject(startEventMetrics) && - typeof startEventMetrics["start"] === "number" && - Number.isFinite(startEventMetrics["start"]) && - isObject(startEventSpanAttributes) && - Array.isArray(startEventSpanParents) && - startEventSpanParents.every( - (parent): parent is string => typeof parent === "string", - ) - ? { - created: startEventCreated, - metrics: { start: startEventMetrics["start"] }, - span_attributes: { ...startEventSpanAttributes }, - span_parents: [...startEventSpanParents], - } - : undefined; - return typeof exported === "string" && - typeof rootSpanId === "string" && - typeof rowId === "string" && - typeof spanId === "string" - ? [ - { - ...(typeof endTime === "number" && Number.isFinite(endTime) - ? { endTime } - : {}), - exported, - rootSpanId, - rowId, - spanId, - ...(normalizedStartEvent - ? { startEvent: normalizedStartEvent } - : {}), - }, - ] - : []; - }) - .slice(-MAX_STORED_SPAN_REFERENCES) - : []; - const llmInputs = Array.isArray(state["llmInputs"]) - ? state["llmInputs"] - .flatMap((entry): EveTraceState["llmInputs"] => { - if (!isObject(entry)) { - return []; - } - const key = entry["key"]; - const input = entry["input"]; - return typeof key === "string" && isCapturedModelInput(input) - ? [{ input, key }] - : []; - }) - .slice(-MAX_STORED_LLM_INPUTS) - : []; - const reasoningBlocks = Array.isArray(state["reasoningBlocks"]) - ? state["reasoningBlocks"] - .flatMap((entry): EveTraceState["reasoningBlocks"] => { - if (!isObject(entry)) { - return []; - } - const content = entry["content"]; - const eventAt = entry["eventAt"]; - const key = entry["key"]; - return typeof content === "string" && - (eventAt === undefined || typeof eventAt === "string") && - typeof key === "string" - ? [ - { - content, - ...(typeof eventAt === "string" ? { eventAt } : {}), - key, - }, - ] - : []; - }) - .slice(-MAX_STORED_REASONING_BLOCKS) - : []; - const stepStarts = Array.isArray(state["stepStarts"]) - ? state["stepStarts"] - .flatMap((entry): EveTraceState["stepStarts"] => { - if (!isObject(entry)) { - return []; - } - const ordinal = entry["ordinal"]; - const open = entry["open"]; - const stepIndex = entry["stepIndex"]; - const turnId = entry["turnId"]; - return typeof ordinal === "number" && - Number.isInteger(ordinal) && - ordinal >= 0 && - typeof open === "boolean" && - typeof stepIndex === "number" && - Number.isInteger(stepIndex) && - typeof turnId === "string" - ? [{ open, ordinal, stepIndex, turnId }] - : []; - }) - .slice(-MAX_STORED_STEP_STARTS) - : []; - return { - llmInputs, - metadata: { ...metadata }, - reasoningBlocks, - spanReferences, - stepStarts, - }; -} - -function readEveTraceState( - state: EveStateHandle, -): EveTraceState { - try { - return normalizeEveTraceState(state.get()); - } catch { - return emptyEveTraceState(); - } -} - -function storeEveReasoning( - state: EveStateHandle, - sessionId: string, - event: Extract, -): readonly EveReasoningBlock[] { - const eventAt = event.meta?.at; - const key = llmInputKey(sessionId, event.data.turnId, event.data.stepIndex); - let stored: readonly EveReasoningBlock[] = []; - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const alreadyStored = normalized.reasoningBlocks.some( - (entry) => - entry.content === event.data.reasoning && - entry.eventAt === eventAt && - entry.key === key, - ); - const reasoningBlocks = alreadyStored - ? normalized.reasoningBlocks - : [ - ...normalized.reasoningBlocks, - { - content: event.data.reasoning, - ...(eventAt ? { eventAt } : {}), - key, - }, - ].slice(-MAX_STORED_REASONING_BLOCKS); - stored = reasoningBlocks.flatMap((entry) => - entry.key === key - ? [ - { - content: entry.content, - ...(entry.eventAt ? { eventAt: entry.eventAt } : {}), - }, - ] - : [], - ); - return alreadyStored ? normalized : { ...normalized, reasoningBlocks }; - }); - return stored; -} - -function readStoredEveReasoning( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): readonly EveReasoningBlock[] { - const key = llmInputKey(sessionId, turnId, stepIndex); - return readEveTraceState(state).reasoningBlocks.flatMap((entry) => - entry.key === key - ? [ - { - content: entry.content, - ...(entry.eventAt ? { eventAt: entry.eventAt } : {}), - }, - ] - : [], - ); -} - -function clearStoredEveReasoning( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): void { - const key = llmInputKey(sessionId, turnId, stepIndex); - state.update((current) => { - const normalized = normalizeEveTraceState(current); - return { - ...normalized, - reasoningBlocks: normalized.reasoningBlocks.filter( - (entry) => entry.key !== key, - ), - }; - }); -} - -function eveOutputMessage(output: unknown): unknown { - return Array.isArray(output) && isObject(output[0]) - ? output[0]["message"] - : undefined; -} - -function mergeEveReasoning( - output: unknown, - reasoning: readonly { content: string }[], -): unknown | undefined { - if (reasoning.length === 0) { - return output; - } - - const choice = Array.isArray(output) && isObject(output[0]) ? output[0] : {}; - const message = isObject(choice["message"]) ? choice["message"] : {}; - return [ - { - ...choice, - index: typeof choice["index"] === "number" ? choice["index"] : 0, - message: { - ...message, - content: "content" in message ? message["content"] : null, - reasoning: reasoning.map((block) => ({ content: block.content })), - role: - typeof message["role"] === "string" ? message["role"] : "assistant", - }, - }, - ]; -} - -function captureEveModelInput( - state: EveStateHandle, - input: EveInstrumentationStepStartedEventInput, -): void { - const sessionId = input.session.id; - const turnId = input.turn.id; - const stepIndex = input.step.index; - const captured = capturedModelInput(input.modelInput); - if (!captured) { - return; - } - - const key = llmInputKey(sessionId, turnId, stepIndex); - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const llmInputs = [...normalized.llmInputs, { input: captured, key }]; - return { - ...normalized, - llmInputs: llmInputs.slice(-MAX_STORED_LLM_INPUTS), - }; - }); -} - -function consumeCapturedEveModelInput( - state: EveStateHandle, - sessionId: string, - turnId: string, - stepIndex: number, -): CapturedEveModelInput | undefined { - try { - const key = llmInputKey(sessionId, turnId, stepIndex); - let input: CapturedEveModelInput | undefined; - state.update((current) => { - const normalized = normalizeEveTraceState(current); - const index = normalized.llmInputs.findIndex( - (candidate) => candidate.key === key, - ); - if (index < 0) { - return normalized; - } - input = normalized.llmInputs[index]?.input; - return { - ...normalized, - llmInputs: normalized.llmInputs.filter( - (_, candidateIndex) => candidateIndex !== index, - ), - }; - }); - return input; - } catch (error) { - debugLogger.warn("Error in Eve LLM input consumption:", error); - return undefined; - } -} - -function capturedModelInput( - modelInput: EveInstrumentationModelInput, -): CapturedEveModelInput | undefined { - const { instructions, messages } = modelInput; - const value: CapturedEveModelMessage[] = []; - if (typeof instructions === "string") { - value.push({ content: instructions, role: "system" }); - } else if (instructions) { - value.push(...instructions.map(capturedEveModelMessage)); - } - value.push(...messages.map(capturedEveModelMessage)); - - try { - const cloned: unknown = JSON.parse(JSON.stringify(value)); - if (!Array.isArray(cloned)) { - return undefined; - } - return cloned; - } catch { - return undefined; - } -} - -function capturedEveModelMessage( - message: EveModelMessage, -): CapturedEveModelMessage { - const { content, role } = message; - if (typeof content === "string") { - return { content, role }; - } - return { content: content.map(capturedEveModelContentPart), role }; -} - -function capturedEveModelContentPart( - part: EveModelMessageContentPart, -): Record { - switch (part.type) { - case "text": - case "reasoning": - return { text: part.text, type: part.type }; - case "image": - return { - image: part.image, - ...(part.mediaType !== undefined ? { mediaType: part.mediaType } : {}), - type: "image", - }; - case "file": - case "reasoning-file": - return { - data: part.data, - ...(part.type === "file" && part.filename !== undefined - ? { filename: part.filename } - : {}), - mediaType: part.mediaType, - type: part.type, - }; - case "custom": - return { - ...("kind" in part ? { kind: part.kind } : {}), - type: "custom", - }; - case "tool-call": - return { - input: part.input, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } - : {}), - toolCallId: part.toolCallId, - toolName: part.toolName, - type: "tool-call", - }; - case "tool-result": { - const output = part.output; - let capturedOutput: Record; - switch (output.type) { - case "text": - case "error-text": - capturedOutput = { type: output.type, value: output.value }; - break; - case "json": - case "error-json": - capturedOutput = { type: output.type, value: output.value }; - break; - case "execution-denied": - capturedOutput = { - ...(output.reason !== undefined ? { reason: output.reason } : {}), - type: "execution-denied", - }; - break; - case "content": - capturedOutput = { - type: "content", - value: output.value.map(capturedEveModelContentPart), - }; - break; - } - return { - output: capturedOutput, - toolCallId: part.toolCallId, - toolName: part.toolName, - type: "tool-result", - }; - } - case "tool-approval-request": - return { - approvalId: part.approvalId, - ...(part.isAutomatic !== undefined - ? { isAutomatic: part.isAutomatic } - : {}), - ...(part.signature !== undefined ? { signature: part.signature } : {}), - toolCallId: part.toolCallId, - type: "tool-approval-request", - }; - case "tool-approval-response": - return { - approvalId: part.approvalId, - approved: part.approved, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } - : {}), - ...(part.reason !== undefined ? { reason: part.reason } : {}), - type: "tool-approval-response", - }; - case "file-data": - case "image-data": - return { - data: part.data, - ...(part.type === "file-data" && part.filename !== undefined - ? { filename: part.filename } - : {}), - mediaType: part.mediaType, - type: part.type, - }; - case "file-url": - case "image-url": - return { - ...(part.type === "file-url" && part.mediaType !== undefined - ? { mediaType: part.mediaType } - : {}), - type: part.type, - url: part.url, - }; - case "file-id": - case "image-file-id": - return { fileId: part.fileId, type: part.type }; - case "file-reference": - case "image-file-reference": - return { - providerReference: part.providerReference, - type: part.type, - }; - } -} - -function isCapturedModelInput(input: unknown): input is CapturedEveModelInput { - return ( - Array.isArray(input) && - input.every( - (message) => - isObject(message) && - (message["role"] === "system" || - message["role"] === "user" || - message["role"] === "assistant" || - message["role"] === "tool") && - (typeof message["content"] === "string" || - (Array.isArray(message["content"]) && - message["content"].every(isObject))), - ) - ); -} - -function llmInputKey( - sessionId: string, - turnId: string, - stepIndex: number, -): string { - return `${sessionId}\0${turnId}\0${stepIndex}`; -} - -function modelMetadataFromRuntime(runtime: unknown): Record { - if (!isObject(runtime)) { - return {}; - } - const modelId = runtime["modelId"]; - return typeof modelId === "string" && !modelId.trim().startsWith("dynamic:") - ? modelMetadataFromModelId(modelId) - : {}; -} - -function modelMetadataFromModelId(modelId: string): Record { - const normalized = modelId.trim(); - if (!normalized) { - return {}; - } - - const slashIndex = normalized.indexOf("/"); - if (slashIndex > 0 && slashIndex < normalized.length - 1) { - return { - model: normalized.slice(slashIndex + 1), - provider: normalized.slice(0, slashIndex), - }; - } - - return { - model: normalized, - }; -} - -function sessionIdFromContext(ctx: unknown): string | undefined { - if (!isObject(ctx)) { - return undefined; - } - const session = ctx["session"]; - if (!isObject(session)) { - return undefined; - } - const id = session["id"]; - return typeof id === "string" ? id : undefined; -} - -function toolMetadataFromTurn(turn: TurnState): Record { - const { model: _model, provider: _provider, ...metadata } = turn.metadata; - return metadata; -} - -function isToolCallAction( - action: unknown, -): action is EveRuntimeToolCallActionRequest { - return ( - isObject(action) && - action["kind"] === "tool-call" && - typeof action["callId"] === "string" && - typeof action["toolName"] === "string" && - isObject(action["input"]) - ); -} - -function isLocalSubagentCallAction( - action: unknown, -): action is Extract { - return ( - isObject(action) && - action["kind"] === "subagent-call" && - typeof action["callId"] === "string" && - isObject(action["input"]) - ); -} - -function isTraceableActionRequest( - action: unknown, -): action is - | EveRuntimeToolCallActionRequest - | Extract { - return isToolCallAction(action) || isLocalSubagentCallAction(action); -} - -function isToolResult( - result: unknown, -): result is EveRuntimeToolResultActionResult { - return ( - isObject(result) && - result["kind"] === "tool-result" && - typeof result["callId"] === "string" && - typeof result["toolName"] === "string" - ); -} - -function isSubagentResult( - result: unknown, -): result is Extract { - return ( - isObject(result) && - result["kind"] === "subagent-result" && - typeof result["callId"] === "string" && - typeof result["subagentName"] === "string" - ); -} - -function normalizedFinishReason( - finishReason: EveAssistantStepFinishReason, -): string { - switch (finishReason) { - case "content-filter": - return "content_filter"; - case "tool-calls": - return "tool_calls"; - default: - return finishReason; - } -} - -function errorFromMessage( - message: string, - code: string, - details?: unknown, -): Error { - const error = new Error(`${code}: ${message}`); - if (details !== undefined) { - error.cause = details; - } - return error; -} - -function actionResultError( - error: EveActionResultError | undefined, - output: unknown, -): Error { - if (error) { - return errorFromMessage(error.message, error.code); - } - const result = new Error("Eve action failed"); - if (output !== undefined) { - result.cause = output; - } - return result; -} - -function eventTime(event: { - readonly meta?: { readonly at: string }; -}): number | undefined { - if (!event.meta?.at) { - return undefined; - } - const timestamp = Date.parse(event.meta.at); - return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; -} - -function turnKey(sessionId: string, turnId: string): string { - return `${sessionId}:${turnId}`; -} - -function toolKey(sessionId: string, callId: string): string { - return `${sessionId}:${callId}`; -} - -async function generateEveIds( - kind: EveEntityKind, - ...parts: string[] -): Promise<{ rowId: string; spanId: string }> { - const [rowId, spanId] = await Promise.all([ - deterministicEveId(`eve:row:${kind}`, ...parts), - deterministicEveId(`eve:${kind}`, ...parts), - ]); - return { rowId, spanId }; -} - -async function deterministicEveId(...parts: string[]): Promise { - const data = new TextEncoder().encode( - parts.map((part) => `${part.length}:${part}`).join("\0"), - ); - const digest = await globalThis.crypto.subtle.digest("SHA-256", data); - const bytes = Array.from(new Uint8Array(digest, 0, 16)); - const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")).join(""); - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; -} diff --git a/js/src/instrumentation/plugins/flue-channels.ts b/js/src/instrumentation/plugins/flue-channels.ts deleted file mode 100644 index 308d50aea..000000000 --- a/js/src/instrumentation/plugins/flue-channels.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { channel, defineChannels } from "../core/channel-definitions"; -import { INSTRUMENTATION_NAMES } from "../../span-origin"; -import type { FlueObservableContext } from "../../vendor-sdk-types/flue"; - -export const flueChannels = defineChannels( - "@flue/runtime", - { - createContext: channel<[unknown], FlueObservableContext>({ - channelName: "createFlueContext", - kind: "sync-stream", - }), - }, - { instrumentationName: INSTRUMENTATION_NAMES.FLUE }, -); diff --git a/js/src/instrumentation/plugins/openai-plugin.ts b/js/src/instrumentation/plugins/openai-plugin.ts deleted file mode 100644 index d1ca89dcc..000000000 --- a/js/src/instrumentation/plugins/openai-plugin.ts +++ /dev/null @@ -1,613 +0,0 @@ -import { BasePlugin } from "../core"; -import { - traceAsyncChannel, - traceStreamingChannel, - traceSyncStreamChannel, - unsubscribeAll, -} from "../core/channel-tracing"; -import { Attachment } from "../../logger"; -import { SpanTypeAttribute, isObject } from "../../../util/index"; -import { getCurrentUnixTimestamp } from "../../util"; -import { processInputAttachments } from "../../wrappers/attachment-utils"; -import { openAIChannels } from "./openai-channels"; -import { - BRAINTRUST_CACHED_STREAM_METRIC, - getCachedMetricFromHeaders, - parseMetricsFromUsage, -} from "../../openai-utils"; -import type { - OpenAIChatChoice, - OpenAIChatCompletionChunk, - OpenAIChatLogprobs, - OpenAIResponseStreamEvent, -} from "../../vendor-sdk-types/openai"; - -/** - * Plugin for OpenAI SDK instrumentation. - * - * Handles instrumentation for: - * - Chat completions (streaming and non-streaming) - * - Embeddings - * - Moderations - * - Beta API (parse, stream) - * - Responses API (create, stream, parse, compact) - */ -export class OpenAIPlugin extends BasePlugin { - constructor() { - super(); - } - - protected onEnable(): void { - // Chat Completions - supports streaming - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.chatCompletionsCreate, { - name: "Chat Completion", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return result?.choices; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateChatCompletionChunks, - }), - ); - - // Embeddings - this.unsubscribers.push( - traceAsyncChannel(openAIChannels.embeddingsCreate, { - name: "Embedding", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input, - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - const embedding = result?.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetrics: (result, _startTime, endEvent) => { - return withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - }, - }), - ); - - // Beta Chat Completions Parse - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { - name: "Chat Completion", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return result?.choices; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateChatCompletionChunks, - }), - ); - - // Beta Chat Completions Stream (sync method returning event-based stream) - this.unsubscribers.push( - traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { - name: "Chat Completion", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "openai" }, - }; - }, - }), - ); - - // Moderations - this.unsubscribers.push( - traceAsyncChannel(openAIChannels.moderationsCreate, { - name: "Moderation", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input, - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return result?.results; - }, - extractMetrics: (result, _startTime, endEvent) => { - return withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - }, - }), - ); - - // Responses API - create (supports streaming via stream=true param) - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.responsesCreate, { - name: "openai.responses.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return processImagesInOutput(result?.output); - }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateResponseStreamEvents, - }), - ); - - // Responses API - stream (sync method returning event-based stream) - this.unsubscribers.push( - traceSyncStreamChannel(openAIChannels.responsesStream, { - name: "openai.responses.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractFromEvent: (event) => { - if (event.type !== "response.completed" || !event.response) { - return {}; - } - - const response = event.response; - const data: Record = {}; - - if (response.output !== undefined) { - data.output = processImagesInOutput(response.output); - } - - const { usage: _usage, output: _output, ...metadata } = response; - if (Object.keys(metadata).length > 0) { - data.metadata = metadata; - } - - data.metrics = parseMetricsFromUsage(response.usage); - return data; - }, - }), - ); - - // Responses API - parse - this.unsubscribers.push( - traceStreamingChannel(openAIChannels.responsesParse, { - name: "openai.responses.parse", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return processImagesInOutput(result?.output); - }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateResponseStreamEvents, - }), - ); - - // Responses API - compact - this.unsubscribers.push( - traceAsyncChannel(openAIChannels.responsesCompact, { - name: "openai.responses.compact", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input: processInputAttachments(input), - metadata: { ...metadata, provider: "openai" }, - }; - }, - extractOutput: (result) => { - return processImagesInOutput(result?.output); - }, - extractMetadata: (result) => { - if (!result) { - return undefined; - } - const { output: _output, usage: _usage, ...metadata } = result; - return Object.keys(metadata).length > 0 ? metadata : undefined; - }, - extractMetrics: (result, startTime, endEvent) => { - const metrics = withCachedMetric( - parseMetricsFromUsage(result?.usage), - result, - endEvent, - ); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } -} - -function getCachedMetricFromEndEvent(endEvent: unknown): number | undefined { - if (!isObject(endEvent)) { - return undefined; - } - - const response = (endEvent as Record).response; - if (!isObject(response)) { - return undefined; - } - - const headers = (response as { headers?: unknown }).headers; - if (!headers || typeof (headers as Headers).get !== "function") { - return undefined; - } - - return getCachedMetricFromHeaders(headers as Headers); -} - -function withCachedMetric( - metrics: Record, - result: unknown, - endEvent?: unknown, -): Record { - if (metrics.cached !== undefined) { - return metrics; - } - - const cachedFromEvent = getCachedMetricFromEndEvent(endEvent); - if (cachedFromEvent !== undefined) { - return { - ...metrics, - cached: cachedFromEvent, - }; - } - - if (!isObject(result)) { - return metrics; - } - - const cached = (result as Record)[ - BRAINTRUST_CACHED_STREAM_METRIC - ]; - - if (typeof cached !== "number") { - return metrics; - } - - return { - ...metrics, - cached, - }; -} - -/** - * Process output to convert base64 images to attachments. - * Used for Responses API image generation output. - */ -export function processImagesInOutput(output: any): any { - if (Array.isArray(output)) { - return output.map(processImagesInOutput); - } - - if (isObject(output)) { - if ( - output.type === "image_generation_call" && - output.result && - typeof output.result === "string" - ) { - const fileExtension = output.output_format || "png"; - const contentType = `image/${fileExtension}`; - - const baseFilename = - output.revised_prompt && typeof output.revised_prompt === "string" - ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") - : "generated_image"; - const filename = `${baseFilename}.${fileExtension}`; - - // Convert base64 string to Blob - const binaryString = atob(output.result); - const bytes = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i); - } - const blob = new Blob([bytes], { type: contentType }); - - const attachment = new Attachment({ - data: blob, - filename: filename, - contentType: contentType, - }); - - return { - ...output, - result: attachment, - }; - } - } - - return output; -} - -function mergeLogprobTokens( - existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], - incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], -): OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"] { - if (incoming === undefined) { - return existing; - } - - if (incoming === null) { - return existing ?? null; - } - - if (Array.isArray(existing)) { - return [...existing, ...incoming]; - } - - return [...incoming]; -} - -function aggregateChatLogprobs( - existing: OpenAIChatLogprobs | null | undefined, - incoming: OpenAIChatLogprobs | null | undefined, -): OpenAIChatLogprobs | null | undefined { - if (incoming === undefined) { - return existing; - } - - if (incoming === null) { - return existing ?? null; - } - - const aggregated: OpenAIChatLogprobs = - existing && existing !== null - ? { ...existing, ...incoming } - : { ...incoming }; - - const content = mergeLogprobTokens(existing?.content, incoming.content); - if (content !== undefined) { - aggregated.content = content; - } - - const refusal = mergeLogprobTokens(existing?.refusal, incoming.refusal); - if (refusal !== undefined) { - aggregated.refusal = refusal; - } - - return aggregated; -} - -/** - * Aggregate chat completion chunks into a single response. - * Combines role (first), content (concatenated), tool_calls (by id), - * finish_reason (last), and usage (last chunk). - */ -export function aggregateChatCompletionChunks( - chunks: OpenAIChatCompletionChunk[], - streamResult?: unknown, - endEvent?: unknown, -): { - output: OpenAIChatChoice[]; - metrics: Record; -} { - let role = undefined; - let content = undefined; - let refusal = undefined; - let tool_calls = undefined; - let logprobs: OpenAIChatLogprobs | null | undefined = undefined; - let finish_reason = undefined; - let metrics: Record = {}; - - for (const chunk of chunks) { - if (chunk.usage) { - metrics = { - ...metrics, - ...parseMetricsFromUsage(chunk.usage), - }; - } - - const choice = chunk.choices?.[0]; - if (!choice) { - continue; - } - - if (choice.finish_reason) { - finish_reason = choice.finish_reason; - } - - logprobs = aggregateChatLogprobs(logprobs, choice.logprobs); - - const delta = choice.delta; - if (!delta) { - continue; - } - - if (delta.finish_reason) { - finish_reason = delta.finish_reason; - } - - if (!role && delta.role) { - role = delta.role; - } - - if (delta.content) { - content = (content || "") + delta.content; - } - - if (delta.refusal) { - refusal = (refusal || "") + delta.refusal; - } - - if (delta.tool_calls) { - const toolDelta = delta.tool_calls[0]; - if ( - !tool_calls || - (toolDelta.id && tool_calls[tool_calls.length - 1].id !== toolDelta.id) - ) { - tool_calls = [ - ...(tool_calls || []), - { - id: toolDelta.id, - type: toolDelta.type, - function: toolDelta.function, - }, - ]; - } else { - tool_calls[tool_calls.length - 1].function.arguments += - toolDelta.function.arguments; - } - } - } - - metrics = withCachedMetric(metrics, streamResult, endEvent); - - return { - metrics, - output: [ - { - index: 0, - message: { - role, - content, - ...(refusal !== undefined ? { refusal } : {}), - tool_calls, - }, - logprobs: logprobs ?? null, - finish_reason, - }, - ], - }; -} - -function aggregateResponseStreamEvents( - chunks: OpenAIResponseStreamEvent[], - _streamResult?: unknown, - endEvent?: unknown, -): { - output: any; - metrics: Record; - metadata?: Record; -} { - let output: any = undefined; - let metrics: Record = {}; - let metadata: Record | undefined = undefined; - - for (const chunk of chunks) { - if (!chunk || !chunk.type || !chunk.response) { - continue; - } - if (chunk.type !== "response.completed") { - continue; - } - - const response = chunk.response; - if (response?.output !== undefined) { - output = processImagesInOutput(response.output); - } - - const { usage: _usage, output: _output, ...rest } = response || {}; - if (Object.keys(rest).length > 0) { - metadata = rest; - } - - metrics = parseMetricsFromUsage(response?.usage); - } - - return { - output, - metrics: withCachedMetric(metrics, undefined, endEvent), - ...(metadata !== undefined ? { metadata } : {}), - }; -} - -export { parseMetricsFromUsage }; diff --git a/js/src/instrumentation/plugins/ai-sdk-channels.ts b/js/src/instrumentation/providers/ai-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/ai-sdk-channels.ts rename to js/src/instrumentation/providers/ai-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts b/js/src/instrumentation/providers/ai-sdk-instrumentation.streaming.test.ts similarity index 92% rename from js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts rename to js/src/instrumentation/providers/ai-sdk-instrumentation.streaming.test.ts index 8803708c3..1921dfa02 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.streaming.test.ts +++ b/js/src/instrumentation/providers/ai-sdk-instrumentation.streaming.test.ts @@ -22,7 +22,6 @@ import { } from "../../wrappers/ai-sdk/harness-agent-context"; import { workflowAgentWrapperSpanCountForTesting } from "../../wrappers/ai-sdk/workflow-agent-context"; import { aiSDKChannels } from "./ai-sdk-channels"; -import { AISDKPlugin } from "./ai-sdk-plugin"; try { configureNode(); @@ -41,7 +40,7 @@ describe("AI SDK streaming instrumentation", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "ai-sdk-plugin.streaming.test.ts", + projectName: "ai-sdk-instrumentation.streaming.test.ts", projectId: "test-project-id", }); }); @@ -890,144 +889,130 @@ describe("AI SDK streaming instrumentation", () => { test("baseStream patch preserves derived stream getters", async () => { expect(await backgroundLogger.drain()).toHaveLength(0); - const plugin = new AISDKPlugin(); - plugin.enable(); - - try { - let chunkSent = false; - const result = (await aiSDKChannels.streamText.tracePromise( - async () => { - const resultRecord = { - baseStream: new ReadableStream({ - pull(controller) { - if (chunkSent) { - controller.close(); - return; - } - - chunkSent = true; - controller.enqueue({ - type: "text-delta", - id: "text-1", - delta: "fresh", - }); - }, - }), - text: Promise.resolve("fresh"), - } as any; - - Object.defineProperty(resultRecord, "textStream", { - configurable: true, - enumerable: true, - get() { - const [textBranch, baseBranch] = this.baseStream.tee(); - this.baseStream = baseBranch; - return textBranch.pipeThrough( - new TransformStream({ - transform(chunk: any, controller) { - if (chunk.type === "text-delta") { - controller.enqueue(chunk.delta); - } - }, - }), - ); - }, - }); + let chunkSent = false; + const result = (await aiSDKChannels.streamText.tracePromise( + async () => { + const resultRecord = { + baseStream: new ReadableStream({ + pull(controller) { + if (chunkSent) { + controller.close(); + return; + } - return resultRecord; - }, - { - arguments: [ - { - model: "mock-stream-model", - prompt: "Reply with fresh.", + chunkSent = true; + controller.enqueue({ + type: "text-delta", + id: "text-1", + delta: "fresh", + }); }, - ], - } as any, - )) as any; + }), + text: Promise.resolve("fresh"), + } as any; + + Object.defineProperty(resultRecord, "textStream", { + configurable: true, + enumerable: true, + get() { + const [textBranch, baseBranch] = this.baseStream.tee(); + this.baseStream = baseBranch; + return textBranch.pipeThrough( + new TransformStream({ + transform(chunk: any, controller) { + if (chunk.type === "text-delta") { + controller.enqueue(chunk.delta); + } + }, + }), + ); + }, + }); - expect( - Object.getOwnPropertyDescriptor(result, "textStream")?.get, - ).toEqual(expect.any(Function)); + return resultRecord; + }, + { + arguments: [ + { + model: "mock-stream-model", + prompt: "Reply with fresh.", + }, + ], + } as any, + )) as any; - let firstText = ""; - for await (const chunk of result.textStream) { - firstText += chunk; - } + expect(Object.getOwnPropertyDescriptor(result, "textStream")?.get).toEqual( + expect.any(Function), + ); - let secondText = ""; - for await (const chunk of result.textStream) { - secondText += chunk; - } + let firstText = ""; + for await (const chunk of result.textStream) { + firstText += chunk; + } - expect(firstText).toBe("fresh"); - expect(secondText).toBe("fresh"); - } finally { - plugin.disable(); + let secondText = ""; + for await (const chunk of result.textStream) { + secondText += chunk; } + + expect(firstText).toBe("fresh"); + expect(secondText).toBe("fresh"); }); test("async iterable stream accessors preserve ReadableStream methods", async () => { expect(await backgroundLogger.drain()).toHaveLength(0); - const plugin = new AISDKPlugin(); - plugin.enable(); - - try { - const result = (await aiSDKChannels.streamText.tracePromise( - async () => { - const resultRecord = { - stream: new ReadableStream({ - start(controller) { - controller.enqueue("v7"); - controller.close(); - }, - }), - text: Promise.resolve("v7"), - } as any; - - Object.defineProperty(resultRecord, "textStream", { - configurable: true, - enumerable: true, - get() { - return this.stream.pipeThrough( - new TransformStream({ - transform(chunk: string, controller) { - controller.enqueue(chunk.toUpperCase()); - }, - }), - ); + const result = (await aiSDKChannels.streamText.tracePromise( + async () => { + const resultRecord = { + stream: new ReadableStream({ + start(controller) { + controller.enqueue("v7"); + controller.close(); }, - }); + }), + text: Promise.resolve("v7"), + } as any; + + Object.defineProperty(resultRecord, "textStream", { + configurable: true, + enumerable: true, + get() { + return this.stream.pipeThrough( + new TransformStream({ + transform(chunk: string, controller) { + controller.enqueue(chunk.toUpperCase()); + }, + }), + ); + }, + }); - return resultRecord; - }, - { - arguments: [ - { - model: "mock-v7-stream-model", - prompt: "Reply with v7.", - }, - ], - } as any, - )) as any; + return resultRecord; + }, + { + arguments: [ + { + model: "mock-v7-stream-model", + prompt: "Reply with v7.", + }, + ], + } as any, + )) as any; - expect(result.stream.pipeThrough).toEqual(expect.any(Function)); - expect(result.stream.getReader).toEqual(expect.any(Function)); + expect(result.stream.pipeThrough).toEqual(expect.any(Function)); + expect(result.stream.getReader).toEqual(expect.any(Function)); - const textStream = result.textStream; - expect(textStream.pipeThrough).toEqual(expect.any(Function)); - expect(textStream.getReader).toEqual(expect.any(Function)); + const textStream = result.textStream; + expect(textStream.pipeThrough).toEqual(expect.any(Function)); + expect(textStream.getReader).toEqual(expect.any(Function)); - const reader = textStream.getReader(); - const first = await reader.read(); - const second = await reader.read(); + const reader = textStream.getReader(); + const first = await reader.read(); + const second = await reader.read(); - expect(first).toEqual({ done: false, value: "V7" }); - expect(second).toEqual({ done: true, value: undefined }); - } finally { - plugin.disable(); - } + expect(first).toEqual({ done: false, value: "V7" }); + expect(second).toEqual({ done: true, value: undefined }); }); test("wrapAgentClass instruments all HarnessAgent turn methods without serializing sessions", async () => { diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts b/js/src/instrumentation/providers/ai-sdk-instrumentation.test.ts similarity index 96% rename from js/src/instrumentation/plugins/ai-sdk-plugin.test.ts rename to js/src/instrumentation/providers/ai-sdk-instrumentation.test.ts index fcc538deb..60fcbc4bf 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/ai-sdk-instrumentation.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, vi, beforeEach } from "vitest"; const telemetryMocks = vi.hoisted(() => ({ braintrustAISDKTelemetry: vi.fn(), @@ -23,13 +23,13 @@ vi.mock("../../wrappers/ai-sdk/telemetry", () => ({ })); import { - AISDKPlugin, + registerAISDKInstrumentation, DEFAULT_DENY_OUTPUT_PATHS, processAISDKCallInput, processAISDKWorkflowAgentCallInput, processAISDKWorkflowAgentModelCallInput, processAISDKOutput as processAISDKOutputActual, -} from "./ai-sdk-plugin"; +} from "./ai-sdk-instrumentation"; import iso from "../../isomorph"; import { serializeAISDKToolsForLogging } from "../../wrappers/ai-sdk/tool-serialization"; import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; @@ -39,7 +39,6 @@ type MockTracingChannel = { handlers: any[]; hasSubscribers: boolean; subscribe: ReturnType; - unsubscribe: ReturnType; }; const mockChannels = new Map(); @@ -47,9 +46,7 @@ const mockChannels = new Map(); // Since these are private, we'll test them through the public API // But we'll also add some tests for the exported utility functions -describe("AISDKPlugin", () => { - let plugin: AISDKPlugin; - +describe("registerAISDKInstrumentation", () => { beforeEach(() => { mockChannels.clear(); telemetryMocks.telemetry = { @@ -70,39 +67,10 @@ describe("AISDKPlugin", () => { channel.handlers.push(handlers); channel.hasSubscribers = true; }), - unsubscribe: vi.fn((handlers: any) => { - channel.handlers = channel.handlers.filter( - (candidate) => candidate !== handlers, - ); - channel.hasSubscribers = channel.handlers.length > 0; - return true; - }), }; mockChannels.set(name, channel); return channel; }); - - plugin = new AISDKPlugin(); - }); - - afterEach(() => { - if (plugin) { - plugin.disable(); - } - }); - - describe("constructor", () => { - it("should create plugin with default config", () => { - const defaultPlugin = new AISDKPlugin(); - expect(defaultPlugin).toBeInstanceOf(AISDKPlugin); - }); - - it("should create plugin with custom config", () => { - const customPlugin = new AISDKPlugin({ - denyOutputPaths: ["custom.path"], - }); - expect(customPlugin).toBeInstanceOf(AISDKPlugin); - }); }); describe("WorkflowAgent input extraction", () => { @@ -192,33 +160,9 @@ describe("AISDKPlugin", () => { }); }); - describe("enable/disable", () => { - it("should enable plugin", () => { - expect(() => plugin.enable()).not.toThrow(); - }); - - it("should disable plugin", () => { - plugin.enable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("should handle multiple enable calls", () => { - plugin.enable(); - expect(() => plugin.enable()).not.toThrow(); - }); - - it("should handle multiple disable calls", () => { - plugin.enable(); - plugin.disable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("should unsubscribe from channels on disable", () => { - plugin.enable(); - plugin.disable(); - // Verify that unsubscribers were called - // This is tested indirectly - if it doesn't throw, unsubscribe worked - expect(true).toBe(true); + describe("registration", () => { + it("registers without throwing", () => { + expect(() => registerAISDKInstrumentation()).not.toThrow(); }); }); @@ -238,7 +182,7 @@ describe("AISDKPlugin", () => { onStepEnd: existingOnStepEnd, }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -309,7 +253,7 @@ describe("AISDKPlugin", () => { onStart: existingOnStart, }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -368,7 +312,7 @@ describe("AISDKPlugin", () => { onStart: vi.fn(), }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -432,7 +376,7 @@ describe("AISDKPlugin", () => { onStart: existingOnStart, }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", @@ -460,7 +404,7 @@ describe("AISDKPlugin", () => { onStart: vi.fn(), }; - plugin.enable(); + registerAISDKInstrumentation(); const channel = mockChannels.get( "orchestrion:ai:createTelemetryDispatcher", diff --git a/js/src/instrumentation/plugins/ai-sdk-plugin.ts b/js/src/instrumentation/providers/ai-sdk-instrumentation.ts similarity index 85% rename from js/src/instrumentation/plugins/ai-sdk-plugin.ts rename to js/src/instrumentation/providers/ai-sdk-instrumentation.ts index e6cbf92f4..8b797e987 100644 --- a/js/src/instrumentation/plugins/ai-sdk-plugin.ts +++ b/js/src/instrumentation/providers/ai-sdk-instrumentation.ts @@ -1,10 +1,9 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { debugLogger } from "../../debug-logger"; import { traceAsyncChannel, traceStreamingChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; @@ -75,7 +74,7 @@ import type { } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; import { BRAINTRUST_AI_SDK_V7_OPERATION_KEY as AI_SDK_V7_OPERATION_KEY } from "../../vendor-sdk-types/ai-sdk-v7-telemetry"; -interface AISDKPluginConfig { +interface AISDKInstrumentationConfig { /** * List of JSON paths to remove from output field. * Uses dot notation with array wildcards: "roundtrips[].request.body" @@ -150,10 +149,10 @@ const AI_SDK_V7_TELEMETRY_CALLBACKS = [ ] as const; /** - * AI SDK plugin that subscribes to instrumentation channels + * Internal AI SDK consumer that subscribes to instrumentation channels * and creates Braintrust spans. * - * This plugin handles: + * This consumer handles: * - generateText (async function) * - streamText (function returning stream) * - generateObject (async function) @@ -167,500 +166,455 @@ const AI_SDK_V7_TELEMETRY_CALLBACKS = [ * - ToolLoopAgent.stream (async method returning stream) * - WorkflowAgent.stream (async method returning stream) * - * The plugin automatically extracts: + * The consumer automatically extracts: * - Model and provider information * - Token usage metrics * - Tool calls and structured outputs * - Streaming responses with time-to-first-token */ -export class AISDKPlugin extends BasePlugin { - private config: AISDKPluginConfig; +class AISDKInstrumentationConsumer { + private config: AISDKInstrumentationConfig; - constructor(config: AISDKPluginConfig = {}) { - super(); + constructor(config: AISDKInstrumentationConfig = {}) { this.config = config; } - protected onEnable(): void { + public register(): void { this.subscribeToAISDK(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToAISDK(): void { const denyOutputPaths = this.config.denyOutputPaths || DEFAULT_DENY_OUTPUT_PATHS; - this.unsubscribers.push(subscribeToAISDKV7TelemetryDispatcher()); - this.unsubscribers.push(subscribeToHarnessAgentCreateSession()); - this.unsubscribers.push( - subscribeToHarnessContinuation( - harnessAgentChannels.continueGenerate, - denyOutputPaths, - ), - subscribeToHarnessContinuation( - harnessAgentChannels.continueStream, - denyOutputPaths, - ), + subscribeToAISDKV7TelemetryDispatcher(); + subscribeToHarnessAgentCreateSession(); + subscribeToHarnessContinuation( + harnessAgentChannels.continueGenerate, + denyOutputPaths, + ); + subscribeToHarnessContinuation( + harnessAgentChannels.continueStream, + denyOutputPaths, ); // generateText - async function that may return streams - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.generateText, { - name: "generateText", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.generateText, { + name: "generateText", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // streamText - function returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.streamText, { - name: "streamText", - type: SpanTypeAttribute.FUNCTION, - shouldTrace: () => currentCloudflareThinkSpan() === undefined, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.streamText, { + name: "streamText", + type: SpanTypeAttribute.FUNCTION, + shouldTrace: () => currentCloudflareThinkSpan() === undefined, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // streamText - sync function returning stream (v4+, used by auto-hook) - this.unsubscribers.push( - traceSyncStreamChannel(aiSDKChannels.streamTextSync, { - name: "streamText", - type: SpanTypeAttribute.FUNCTION, - shouldTrace: () => currentCloudflareThinkSpan() === undefined, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceSyncStreamChannel(aiSDKChannels.streamTextSync, { + name: "streamText", + type: SpanTypeAttribute.FUNCTION, + shouldTrace: () => currentCloudflareThinkSpan() === undefined, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // generateObject - async function that may return streams - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.generateObject, { - name: "generateObject", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.generateObject, { + name: "generateObject", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // streamObject - function returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.streamObject, { - name: "streamObject", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.streamObject, { + name: "streamObject", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // streamObject - sync function returning stream (v4+, used by auto-hook) - this.unsubscribers.push( - traceSyncStreamChannel(aiSDKChannels.streamObjectSync, { - name: "streamObject", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths), - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceSyncStreamChannel(aiSDKChannels.streamObjectSync, { + name: "streamObject", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths), + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // embed - async embedding function - this.unsubscribers.push( - traceAsyncChannel(aiSDKChannels.embed, { - name: "embed", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event) => - prepareAISDKEmbedInput(params, event.self), - extractOutput: (result, endEvent) => - processAISDKEmbeddingOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - }), - ); + traceAsyncChannel(aiSDKChannels.embed, { + name: "embed", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event) => + prepareAISDKEmbedInput(params, event.self), + extractOutput: (result, endEvent) => + processAISDKEmbeddingOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + }); // embedMany - async embedding batch function - this.unsubscribers.push( - traceAsyncChannel(aiSDKChannels.embedMany, { - name: "embedMany", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event) => - prepareAISDKEmbedInput(params, event.self), - extractOutput: (result, endEvent) => - processAISDKEmbeddingOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - }), - ); + traceAsyncChannel(aiSDKChannels.embedMany, { + name: "embedMany", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event) => + prepareAISDKEmbedInput(params, event.self), + extractOutput: (result, endEvent) => + processAISDKEmbeddingOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + }); // rerank - async reranking function - this.unsubscribers.push( - traceAsyncChannel(aiSDKChannels.rerank, { - name: "rerank", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event) => - prepareAISDKRerankInput(params, event.self), - extractOutput: (result, endEvent) => - processAISDKRerankOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - }), - ); + traceAsyncChannel(aiSDKChannels.rerank, { + name: "rerank", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event) => + prepareAISDKRerankInput(params, event.self), + extractOutput: (result, endEvent) => + processAISDKRerankOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + }); // Agent.generate - async method - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.agentGenerate, { - name: "Agent.generate", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.agentGenerate, { + name: "Agent.generate", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // Agent.stream - async method returning stream (v5, used by wrapAISDK) - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.agentStream, { - name: "Agent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.agentStream, { + name: "Agent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // Agent.stream - sync method returning stream (v5, used by auto-hook) - this.unsubscribers.push( - traceSyncStreamChannel(aiSDKChannels.agentStreamSync, { - name: "Agent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceSyncStreamChannel(aiSDKChannels.agentStreamSync, { + name: "Agent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // HarnessAgent.generate - one task span per agent turn - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.generate, { - name: "HarnessAgent.generate", - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result) => extractTokenMetrics(result), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(harnessAgentChannels.generate, { + name: "HarnessAgent.generate", + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result) => extractTokenMetrics(result), + aggregateChunks: aggregateAISDKChunks, + }); // HarnessAgent.stream - async method returning an AI SDK stream result - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.stream, { - name: "HarnessAgent.stream", - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime) => ({ - ...extractTokenMetrics(result), - ...(startTime === undefined - ? {} - : { - time_to_first_token: getCurrentUnixTimestamp() - startTime, - }), - }), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - resolvePromiseUsage: true, - span, - startTime, - }), + traceStreamingChannel(harnessAgentChannels.stream, { + name: "HarnessAgent.stream", + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime) => ({ + ...extractTokenMetrics(result), + ...(startTime === undefined + ? {} + : { + time_to_first_token: getCurrentUnixTimestamp() - startTime, + }), }), - ); + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + resolvePromiseUsage: true, + span, + startTime, + }), + }); // Trace a continuation as its own task only when its original turn cannot // be recovered. Known continuations extend the original Harness task. - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.continueGenerate, { - name: "HarnessAgent.continueGenerate", - shouldTrace: (args) => - !harnessContinuationParent(harnessSessionFromArguments(args)), - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result) => extractTokenMetrics(result), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(harnessAgentChannels.continueGenerate, { + name: "HarnessAgent.continueGenerate", + shouldTrace: (args) => + !harnessContinuationParent(harnessSessionFromArguments(args)), + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result) => extractTokenMetrics(result), + aggregateChunks: aggregateAISDKChunks, + }); - this.unsubscribers.push( - traceStreamingChannel(harnessAgentChannels.continueStream, { - name: "HarnessAgent.continueStream", - shouldTrace: (args) => - !harnessContinuationParent(harnessSessionFromArguments(args)), - startSpan: _internalStartSpanWithInitialMerge, - type: SpanTypeAttribute.TASK, - extractInput: ([params], event, span) => - prepareAISDKHarnessAgentInput(params, event.self, span), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime) => ({ - ...extractTokenMetrics(result), - ...(startTime === undefined - ? {} - : { - time_to_first_token: getCurrentUnixTimestamp() - startTime, - }), - }), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - resolvePromiseUsage: true, - span, - startTime, - }), + traceStreamingChannel(harnessAgentChannels.continueStream, { + name: "HarnessAgent.continueStream", + shouldTrace: (args) => + !harnessContinuationParent(harnessSessionFromArguments(args)), + startSpan: _internalStartSpanWithInitialMerge, + type: SpanTypeAttribute.TASK, + extractInput: ([params], event, span) => + prepareAISDKHarnessAgentInput(params, event.self, span), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime) => ({ + ...extractTokenMetrics(result), + ...(startTime === undefined + ? {} + : { + time_to_first_token: getCurrentUnixTimestamp() - startTime, + }), }), - ); + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + resolvePromiseUsage: true, + span, + startTime, + }), + }); // ToolLoopAgent.generate - async method - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.toolLoopAgentGenerate, { - name: "ToolLoopAgent.generate", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - }), - ); + traceStreamingChannel(aiSDKChannels.toolLoopAgentGenerate, { + name: "ToolLoopAgent.generate", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + }); // ToolLoopAgent.stream - async method returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.toolLoopAgentStream, { - name: "ToolLoopAgent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKCallInput(params, event, span, denyOutputPaths, { - agentOwner: true, - }), - extractOutput: (result, endEvent) => - processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ), - extractMetrics: (result, startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent, startTime), - aggregateChunks: aggregateAISDKChunks, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.toolLoopAgentStream, { + name: "ToolLoopAgent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKCallInput(params, event, span, denyOutputPaths, { + agentOwner: true, + }), + extractOutput: (result, endEvent) => + processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ), + extractMetrics: (result, startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent, startTime), + aggregateChunks: aggregateAISDKChunks, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + result, + span, + startTime, + }), + }); // WorkflowAgent.stream - async method returning stream - this.unsubscribers.push( - traceStreamingChannel(aiSDKChannels.workflowAgentStream, { - name: "WorkflowAgent.stream", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params], event, span) => - prepareAISDKWorkflowAgentStreamInput( - params, - event, - span, - denyOutputPaths, - ), - extractOutput: (result, endEvent) => { - finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); - return processAISDKOutput( - result, - resolveDenyOutputPaths(endEvent, denyOutputPaths), - ); - }, - extractMetrics: (result, _startTime, endEvent) => - extractTopLevelAISDKMetrics(result, endEvent), - aggregateChunks: aggregateAISDKChunks, - onComplete: ({ span }) => { - unregisterWorkflowAgentWrapperSpan(span); - }, - onError: ({ event, span }) => { - finalizeAISDKChildTracing(event as { [key: string]: unknown }); - unregisterWorkflowAgentWrapperSpan(span); - }, - patchResult: ({ endEvent, result, span, startTime }) => - patchAISDKStreamingResult({ - defaultDenyOutputPaths: denyOutputPaths, - endEvent, - onComplete: () => unregisterWorkflowAgentWrapperSpan(span), - onCancel: () => unregisterWorkflowAgentWrapperSpan(span), - onError: () => unregisterWorkflowAgentWrapperSpan(span), - result, - span, - startTime, - }), - }), - ); + traceStreamingChannel(aiSDKChannels.workflowAgentStream, { + name: "WorkflowAgent.stream", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params], event, span) => + prepareAISDKWorkflowAgentStreamInput( + params, + event, + span, + denyOutputPaths, + ), + extractOutput: (result, endEvent) => { + finalizeAISDKChildTracing(endEvent as { [key: string]: unknown }); + return processAISDKOutput( + result, + resolveDenyOutputPaths(endEvent, denyOutputPaths), + ); + }, + extractMetrics: (result, _startTime, endEvent) => + extractTopLevelAISDKMetrics(result, endEvent), + aggregateChunks: aggregateAISDKChunks, + onComplete: ({ span }) => { + unregisterWorkflowAgentWrapperSpan(span); + }, + onError: ({ event, span }) => { + finalizeAISDKChildTracing(event as { [key: string]: unknown }); + unregisterWorkflowAgentWrapperSpan(span); + }, + patchResult: ({ endEvent, result, span, startTime }) => + patchAISDKStreamingResult({ + defaultDenyOutputPaths: denyOutputPaths, + endEvent, + onComplete: () => unregisterWorkflowAgentWrapperSpan(span), + onCancel: () => unregisterWorkflowAgentWrapperSpan(span), + onError: () => unregisterWorkflowAgentWrapperSpan(span), + result, + span, + startTime, + }), + }); } } -function subscribeToHarnessAgentCreateSession(): () => void { +function subscribeToHarnessAgentCreateSession(): void { const channel = harnessAgentChannels.createSession.tracingChannel(); const parents = new WeakMap(); const handlers: IsoChannelHandlers< @@ -682,7 +636,6 @@ function subscribeToHarnessAgentCreateSession(): () => void { }; channel.subscribe(handlers); - return () => channel.unsubscribe(handlers); } type HarnessContinuationChannel = typeof harnessAgentChannels.continueGenerate; @@ -700,14 +653,11 @@ function harnessContinuationParentFromEvent( function subscribeToHarnessContinuation( continuationChannel: HarnessContinuationChannel, defaultDenyOutputPaths: string[], -): () => void { +): void { const channel = continuationChannel.tracingChannel(); const parents = new WeakMap(); const startTimes = new WeakMap(); - const unbindParentStore = bindHarnessTurnParentToStart( - channel, - harnessContinuationParentFromEvent, - ); + bindHarnessTurnParentToStart(channel, harnessContinuationParentFromEvent); const handlers: IsoChannelHandlers< ChannelMessage > = { @@ -833,13 +783,9 @@ function subscribeToHarnessContinuation( }; channel.subscribe(handlers); - return () => { - unbindParentStore(); - channel.unsubscribe(handlers); - }; } -function subscribeToAISDKV7TelemetryDispatcher(): () => void { +function subscribeToAISDKV7TelemetryDispatcher(): void { const channel = aiSDKChannels.v7CreateTelemetryDispatcher.tracingChannel(); const telemetry = braintrustAISDKTelemetry(); const handlers: IsoChannelHandlers< @@ -860,10 +806,6 @@ function subscribeToAISDKV7TelemetryDispatcher(): () => void { }; channel.subscribe(handlers); - - return () => { - channel.unsubscribe(handlers); - }; } function patchAISDKV7TelemetryDispatcher( @@ -4274,3 +4216,12 @@ function omit( return result; } + +let aiSDKInstrumentationConsumer: AISDKInstrumentationConsumer | undefined; + +export function registerAISDKInstrumentation( + config: AISDKInstrumentationConfig = {}, +): void { + aiSDKInstrumentationConsumer ??= new AISDKInstrumentationConsumer(config); + aiSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/ai-sdk-v7-telemetry.test.ts b/js/src/instrumentation/providers/ai-sdk-v7-telemetry.test.ts similarity index 100% rename from js/src/instrumentation/plugins/ai-sdk-v7-telemetry.test.ts rename to js/src/instrumentation/providers/ai-sdk-v7-telemetry.test.ts diff --git a/js/src/instrumentation/plugins/anthropic-channels.ts b/js/src/instrumentation/providers/anthropic-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/anthropic-channels.ts rename to js/src/instrumentation/providers/anthropic-channels.ts diff --git a/js/src/instrumentation/plugins/anthropic-plugin.test.ts b/js/src/instrumentation/providers/anthropic-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/anthropic-plugin.test.ts rename to js/src/instrumentation/providers/anthropic-instrumentation.test.ts index ffad58aff..3e3d56708 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.test.ts +++ b/js/src/instrumentation/providers/anthropic-instrumentation.test.ts @@ -12,7 +12,7 @@ import { aggregateAnthropicStreamChunks, processAttachmentsInInput, coalesceInput, -} from "./anthropic-plugin"; +} from "./anthropic-instrumentation"; import { Attachment } from "../../logger"; const parseMetricsFromUsageForTest = (usage: unknown) => diff --git a/js/src/instrumentation/plugins/anthropic-plugin.ts b/js/src/instrumentation/providers/anthropic-instrumentation.ts similarity index 98% rename from js/src/instrumentation/plugins/anthropic-plugin.ts rename to js/src/instrumentation/providers/anthropic-instrumentation.ts index 36cad5a5c..9b6cc128e 100644 --- a/js/src/instrumentation/plugins/anthropic-plugin.ts +++ b/js/src/instrumentation/providers/anthropic-instrumentation.ts @@ -1,5 +1,5 @@ -import { BasePlugin, toLoggedError } from "../core"; -import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { toLoggedError } from "../core"; +import { traceStreamingChannel } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import { Attachment, @@ -93,30 +93,26 @@ const ANTHROPIC_TOOL_RUNNER_TOOL_WRAPPED = Symbol.for( ); /** - * Auto-instrumentation plugin for the Anthropic SDK. + * Internal auto-instrumentation consumer for the Anthropic SDK. * - * This plugin subscribes to orchestrion channels for Anthropic SDK methods + * This consumer subscribes to orchestrion channels for Anthropic SDK methods * and creates Braintrust spans to track: * - messages.create (streaming and non-streaming) * - beta.messages.create (streaming and non-streaming) * - * The plugin handles: + * The consumer handles: * - Anthropic-specific token metrics (including cache tokens) * - Processing message streams * - Converting base64 attachments to Attachment objects * - Streaming and non-streaming responses */ -export class AnthropicPlugin extends BasePlugin { - protected onEnable(): void { +class AnthropicInstrumentationConsumer { + public register(): void { this.subscribeToAnthropicChannels(); this.subscribeToAnthropicToolRunner(); this.subscribeToAnthropicSessionStreams(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToAnthropicChannels(): void { const anthropicConfig = { name: "anthropic.messages.create", @@ -163,17 +159,13 @@ export class AnthropicPlugin extends BasePlugin { }; // Messages API - supports streaming via stream=true parameter - this.unsubscribers.push( - traceStreamingChannel(anthropicChannels.messagesCreate, anthropicConfig), - ); + traceStreamingChannel(anthropicChannels.messagesCreate, anthropicConfig); // Beta Messages API - supports streaming via stream=true parameter - this.unsubscribers.push( - traceStreamingChannel(anthropicChannels.betaMessagesCreate, { - ...anthropicConfig, - name: "anthropic.messages.create", - }), - ); + traceStreamingChannel(anthropicChannels.betaMessagesCreate, { + ...anthropicConfig, + name: "anthropic.messages.create", + }); } private subscribeToAnthropicToolRunner(): void { @@ -250,9 +242,6 @@ export class AnthropicPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private subscribeToAnthropicSessionStreams(): void { @@ -305,7 +294,6 @@ export class AnthropicPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => tracingChannel.unsubscribe(handlers)); } } @@ -1686,3 +1674,12 @@ export function coalesceInput( } return input; } + +let anthropicInstrumentationConsumer: + | AnthropicInstrumentationConsumer + | undefined; + +export function registerAnthropicInstrumentation(): void { + anthropicInstrumentationConsumer ??= new AnthropicInstrumentationConsumer(); + anthropicInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/anthropic-sessions-plugin.test.ts b/js/src/instrumentation/providers/anthropic-sessions-instrumentation.test.ts similarity index 97% rename from js/src/instrumentation/plugins/anthropic-sessions-plugin.test.ts rename to js/src/instrumentation/providers/anthropic-sessions-instrumentation.test.ts index 1d4cb48ac..44bd501d6 100644 --- a/js/src/instrumentation/plugins/anthropic-sessions-plugin.test.ts +++ b/js/src/instrumentation/providers/anthropic-sessions-instrumentation.test.ts @@ -28,11 +28,11 @@ vi.mock("../../logger", async (importOriginal) => { import iso from "../../isomorph"; import { collectAnthropicSession } from "../../wrappers/anthropic-session-collector"; -import { AnthropicPlugin } from "./anthropic-plugin"; +import { registerAnthropicInstrumentation } from "./anthropic-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("AnthropicPlugin Sessions instrumentation", () => { +describe("registerAnthropicInstrumentation Sessions instrumentation", () => { let currentSpan: TestSpan | undefined; let handlersByName: Map; let spans: TestSpan[]; @@ -43,7 +43,6 @@ describe("AnthropicPlugin Sessions instrumentation", () => { spans = []; mockNewTracingChannel.mockImplementation((name: string) => ({ subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), - unsubscribe: vi.fn(), })); mockWithCurrent.mockImplementation( (span: TestSpan, callback: () => unknown) => { @@ -78,7 +77,7 @@ describe("AnthropicPlugin Sessions instrumentation", () => { }); it("records a session turn with model and tool children", async () => { - new AnthropicPlugin().enable(); + registerAnthropicInstrumentation(); const handlers = handlersByName.get( "orchestrion:@anthropic-ai/sdk:beta.sessions.events.stream", ); @@ -262,7 +261,7 @@ describe("AnthropicPlugin Sessions instrumentation", () => { }); it("does not record an uncollected session stream", async () => { - new AnthropicPlugin().enable(); + registerAnthropicInstrumentation(); const handlers = handlersByName.get( "orchestrion:@anthropic-ai/sdk:beta.sessions.events.stream", ); @@ -286,7 +285,7 @@ describe("AnthropicPlugin Sessions instrumentation", () => { }); it("uses a distinct task name for thread streams and records denials", async () => { - new AnthropicPlugin().enable(); + registerAnthropicInstrumentation(); const handlers = handlersByName.get( "orchestrion:@anthropic-ai/sdk:beta.sessions.threads.events.stream", ); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-channels.ts b/js/src/instrumentation/providers/bedrock-runtime-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/bedrock-runtime-channels.ts rename to js/src/instrumentation/providers/bedrock-runtime-channels.ts diff --git a/js/src/instrumentation/plugins/bedrock-runtime-common.ts b/js/src/instrumentation/providers/bedrock-runtime-common.ts similarity index 100% rename from js/src/instrumentation/plugins/bedrock-runtime-common.ts rename to js/src/instrumentation/providers/bedrock-runtime-common.ts diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts rename to js/src/instrumentation/providers/bedrock-runtime-instrumentation.test.ts index fc74a9068..b22906083 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.test.ts +++ b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.test.ts @@ -9,7 +9,7 @@ import { aggregateBedrockConverseStreamChunks, aggregateInvokeModelResponseStreamChunks, parseBedrockRuntimeMetrics, -} from "./bedrock-runtime-plugin"; +} from "./bedrock-runtime-instrumentation"; try { configureNode(); @@ -29,7 +29,7 @@ class GetObjectCommand { constructor(public input: Record) {} } -describe("BedrockRuntimePlugin", () => { +describe("registerBedrockRuntimeInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -41,7 +41,7 @@ describe("BedrockRuntimePlugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "bedrock-runtime-plugin.test.ts", + projectName: "bedrock-runtime-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.ts similarity index 97% rename from js/src/instrumentation/plugins/bedrock-runtime-plugin.ts rename to js/src/instrumentation/providers/bedrock-runtime-instrumentation.ts index a759d1ffe..dfd5a47be 100644 --- a/js/src/instrumentation/plugins/bedrock-runtime-plugin.ts +++ b/js/src/instrumentation/providers/bedrock-runtime-instrumentation.ts @@ -1,5 +1,4 @@ -import { BasePlugin } from "../core"; -import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { traceStreamingChannel } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; @@ -25,26 +24,18 @@ import { getBedrockRuntimeOperation, } from "./bedrock-runtime-common"; -export class BedrockRuntimePlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - ...[ - bedrockRuntimeChannels.clientSend, - smithyCoreChannels.clientSend, - smithyClientChannels.clientSend, - ].map((channel) => traceBedrockRuntimeClientSendChannel(channel)), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); +export function registerBedrockRuntimeInstrumentation(): void { + for (const channel of [ + bedrockRuntimeChannels.clientSend, + smithyCoreChannels.clientSend, + smithyClientChannels.clientSend, + ]) { + traceBedrockRuntimeClientSendChannel(channel); } } -function traceBedrockRuntimeClientSendChannel( - channel: AnyAsyncChannel, -): () => void { - return traceStreamingChannel(channel, { +function traceBedrockRuntimeClientSendChannel(channel: AnyAsyncChannel): void { + traceStreamingChannel(channel, { name: ([command]) => buildBedrockRuntimeSpanInfo(command).name, shouldTrace: ([command, optionsOrCb, cb]) => getBedrockRuntimeOperation(command) !== undefined && diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-channels.ts b/js/src/instrumentation/providers/claude-agent-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-channels.ts rename to js/src/instrumentation/providers/claude-agent-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-instrumentation-constants.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation-constants.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-instrumentation-constants.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation-constants.ts diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.streaming.test.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.streaming.test.ts similarity index 99% rename from js/src/instrumentation/plugins/claude-agent-sdk-plugin.streaming.test.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation.streaming.test.ts index 42101e066..ddaeeab1f 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.streaming.test.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.streaming.test.ts @@ -120,7 +120,7 @@ describe("Claude Agent SDK streaming instrumentation", () => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ projectId: "test-project-id", - projectName: "claude-agent-sdk-plugin.streaming.test.ts", + projectName: "claude-agent-sdk-instrumentation.streaming.test.ts", }); }); diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts similarity index 86% rename from js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts index f1d446bcd..93265e4a3 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.test.ts @@ -30,7 +30,7 @@ vi.mock("../core/stream-patcher", () => ({ }), })); -import { ClaudeAgentSDKPlugin } from "./claude-agent-sdk-plugin"; +import { registerClaudeAgentSDKInstrumentation } from "./claude-agent-sdk-instrumentation"; import iso from "../../isomorph"; import { startSpan } from "../../logger"; @@ -72,79 +72,26 @@ vi.mock("../../wrappers/attachment-utils", () => ({ // rules, so these tests run the real implementation rather than a stand-in that // could drift from it. -vi.mock("../core", async (importOriginal) => { - const actual = await importOriginal(); - - return { - ...actual, - BasePlugin: class BasePlugin { - protected enabled = false; - protected unsubscribers: Array<() => void> = []; - - enable(): void { - if (this.enabled) { - return; - } - this.enabled = true; - this.onEnable(); - } - - disable(): void { - if (!this.enabled) { - return; - } - this.enabled = false; - this.onDisable(); - } - - protected onEnable(): void { - // To be implemented by subclass - } - - protected onDisable(): void { - // To be implemented by subclass - } - }, - isAsyncIterable: vi.fn( - (val: unknown) => - val !== null && - typeof val === "object" && - Symbol.asyncIterator in val && - typeof (val as any)[Symbol.asyncIterator] === "function", - ), - patchStreamIfNeeded: vi.fn((stream, _callbacks) => { - // Return the stream unchanged for simple tests - return stream; - }), - }; -}); - -describe("ClaudeAgentSDKPlugin", () => { - let plugin: ClaudeAgentSDKPlugin; +describe("registerClaudeAgentSDKInstrumentation", () => { let mockChannel: any; - let mockUnsubscribe: any; beforeEach(() => { streamPatcherMock.options = undefined; - mockUnsubscribe = vi.fn(); mockChannel = { subscribe: vi.fn(), - unsubscribe: mockUnsubscribe, hasSubscribers: false, }; mockNewTracingChannel.mockReturnValue(mockChannel); - - plugin = new ClaudeAgentSDKPlugin(); }); afterEach(() => { vi.clearAllMocks(); }); - describe("enable", () => { - it("should enable the plugin and subscribe to channels", () => { - plugin.enable(); + describe("registration", () => { + it("subscribes to the channel", () => { + registerClaudeAgentSDKInstrumentation(); expect(mockNewTracingChannel).toHaveBeenCalledWith( "orchestrion:@anthropic-ai/claude-agent-sdk:query", @@ -158,50 +105,13 @@ describe("ClaudeAgentSDKPlugin", () => { }), ); }); - - it("should not subscribe twice if already enabled", () => { - plugin.enable(); - plugin.enable(); - - expect(mockChannel.subscribe).toHaveBeenCalledTimes(1); - }); - - it("should store unsubscribe function", () => { - plugin.enable(); - - expect((plugin as any).unsubscribers).toHaveLength(1); - expect((plugin as any).unsubscribers[0]).toBeInstanceOf(Function); - }); - }); - - describe("disable", () => { - it("should unsubscribe from all channels", () => { - plugin.enable(); - plugin.disable(); - - expect(mockUnsubscribe).toHaveBeenCalledTimes(1); - expect((plugin as any).unsubscribers).toHaveLength(0); - }); - - it("should not unsubscribe if not enabled", () => { - plugin.disable(); - - expect(mockUnsubscribe).not.toHaveBeenCalled(); - }); - - it("should clear unsubscribers array", () => { - plugin.enable(); - plugin.disable(); - - expect((plugin as any).unsubscribers).toHaveLength(0); - }); }); describe("channel subscription handlers", () => { let handlers: any; beforeEach(() => { - plugin.enable(); + registerClaudeAgentSDKInstrumentation(); handlers = mockChannel.subscribe.mock.calls[0][0]; }); @@ -745,24 +655,4 @@ describe("ClaudeAgentSDKPlugin", () => { }); }); }); - - describe("enable/disable lifecycle", () => { - it("should allow re-enabling after disable", () => { - plugin.enable(); - plugin.disable(); - plugin.enable(); - - expect(mockChannel.subscribe).toHaveBeenCalledTimes(2); - }); - - it("should properly clean up on multiple enable/disable cycles", () => { - plugin.enable(); - plugin.disable(); - plugin.enable(); - plugin.disable(); - - expect(mockUnsubscribe).toHaveBeenCalledTimes(2); - expect((plugin as any).unsubscribers).toHaveLength(0); - }); - }); }); diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts similarity index 99% rename from js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts rename to js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts index 96f52c9b6..9944ebe60 100644 --- a/js/src/instrumentation/plugins/claude-agent-sdk-plugin.ts +++ b/js/src/instrumentation/providers/claude-agent-sdk-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; @@ -1578,18 +1577,11 @@ async function finalizeQuerySpan(state: QueryState): Promise { } } -export class ClaudeAgentSDKPlugin extends BasePlugin { - protected onEnable(): void { +class ClaudeAgentSDKInstrumentationConsumer { + public register(): void { this.subscribeToQuery(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToQuery(): void { const channel = claudeAgentSDKChannels.query.tracingChannel(); const spans = new WeakMap(); @@ -1854,8 +1846,15 @@ export class ClaudeAgentSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } } + +let claudeAgentSDKInstrumentationConsumer: + | ClaudeAgentSDKInstrumentationConsumer + | undefined; + +export function registerClaudeAgentSDKInstrumentation(): void { + claudeAgentSDKInstrumentationConsumer ??= + new ClaudeAgentSDKInstrumentationConsumer(); + claudeAgentSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-local-tool-context.ts b/js/src/instrumentation/providers/claude-agent-sdk-local-tool-context.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-local-tool-context.ts rename to js/src/instrumentation/providers/claude-agent-sdk-local-tool-context.ts diff --git a/js/src/instrumentation/plugins/claude-agent-sdk-local-tool-spans.ts b/js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts similarity index 100% rename from js/src/instrumentation/plugins/claude-agent-sdk-local-tool-spans.ts rename to js/src/instrumentation/providers/claude-agent-sdk-local-tool-spans.ts diff --git a/js/src/instrumentation/plugins/cloudflare-agents-channels.ts b/js/src/instrumentation/providers/cloudflare-agents-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-agents-channels.ts rename to js/src/instrumentation/providers/cloudflare-agents-channels.ts diff --git a/js/src/instrumentation/plugins/cloudflare-agents-plugin.test.ts b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.test.ts similarity index 88% rename from js/src/instrumentation/plugins/cloudflare-agents-plugin.test.ts rename to js/src/instrumentation/providers/cloudflare-agents-instrumentation.test.ts index d6d2de7de..970297b01 100644 --- a/js/src/instrumentation/plugins/cloudflare-agents-plugin.test.ts +++ b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.test.ts @@ -29,14 +29,13 @@ vi.mock("../../isomorph", () => ({ })); import iso from "../../isomorph"; -import { CloudflareAgentsPlugin } from "./cloudflare-agents-plugin"; +import { registerCloudflareAgentsInstrumentation } from "./cloudflare-agents-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("CloudflareAgentsPlugin", () => { +describe("registerCloudflareAgentsInstrumentation", () => { let handlers: any; let subscribe: ReturnType; - let unsubscribe: ReturnType; let spans: Array<{ args: any; context: any; @@ -49,8 +48,7 @@ describe("CloudflareAgentsPlugin", () => { subscribe = vi.fn((nextHandlers) => { handlers = nextHandlers; }); - unsubscribe = vi.fn(); - mockNewTracingChannel.mockReturnValue({ subscribe, unsubscribe }); + mockNewTracingChannel.mockReturnValue({ subscribe }); mockStartSpan.mockImplementation((args: any, context: any) => { const span = { args, context, end: vi.fn(), log: vi.fn() }; spans.push(span); @@ -62,19 +60,13 @@ describe("CloudflareAgentsPlugin", () => { vi.clearAllMocks(); }); - it("subscribes idempotently to Agent.runAgentTool", () => { - const plugin = new CloudflareAgentsPlugin(); - plugin.enable(); - plugin.enable(); + it("subscribes to the process-lifetime channel", () => { + registerCloudflareAgentsInstrumentation(); expect(mockNewTracingChannel).toHaveBeenCalledWith( "orchestrion:agents:Agent.runAgentTool", ); expect(subscribe).toHaveBeenCalledTimes(1); - - plugin.disable(); - plugin.disable(); - expect(unsubscribe).toHaveBeenCalledTimes(1); }); it("keeps SDK-controlled context out of the public start-span arguments", () => { @@ -86,7 +78,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("records only the child class name, input, and completed output", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); class ResearchAgent {} const event = { arguments: [ @@ -142,7 +134,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("records returned terminal error strings", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); class FailingAgent {} const event = { arguments: [FailingAgent, { input: "fail" }] }; @@ -164,7 +156,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("records the original rejection and preserves concurrent span state", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); class FirstAgent {} class SecondAgent {} const first = { arguments: [FirstAgent, { input: 1 }] }; @@ -188,7 +180,7 @@ describe("CloudflareAgentsPlugin", () => { }); it("skips detached runs and does not invoke getters", () => { - new CloudflareAgentsPlugin().enable(); + registerCloudflareAgentsInstrumentation(); const nameGetter = vi.fn(() => "GetterAgent"); const inputGetter = vi.fn(() => "getter-input"); const AgentWithGetter = Object.defineProperty(function () {}, "name", { diff --git a/js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts new file mode 100644 index 000000000..36fe3671a --- /dev/null +++ b/js/src/instrumentation/providers/cloudflare-agents-instrumentation.ts @@ -0,0 +1,129 @@ +import { debugLogger } from "../../debug-logger"; +import type { IsoChannelHandlers } from "../../isomorph"; +import { _internalStartSpanWithContext } from "../../logger"; +import type { Span } from "../../logger"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { SpanTypeAttribute } from "../../../util/index"; +import type { ChannelMessage } from "../core/channel-definitions"; +import { cloudflareAgentsChannels } from "./cloudflare-agents-channels"; + +const CLOUDFLARE_WORKERS_CONTEXT = { + span_origin: { + environment: { type: "server", name: "cloudflare_workers" }, + }, +}; + +export function registerCloudflareAgentsInstrumentation(): void { + const channel = cloudflareAgentsChannels.runAgentTool.tracingChannel(); + const spans = new WeakMap(); + const handlers: IsoChannelHandlers< + ChannelMessage + > = { + start: (event) => { + try { + const agentClass = event.arguments[0]; + const options = event.arguments[1]; + if (ownValue(options, "detached")) { + return; + } + + const name = ownValue(agentClass, "name"); + if (typeof name !== "string" || name.length === 0) { + debugLogger.warn( + "Skipping Cloudflare Agents runAgentTool span because the child agent class has no name.", + ); + return; + } + + const span = _internalStartSpanWithContext( + withSpanInstrumentationName( + { + name, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + event: { + input: ownValue(options, "input"), + }, + }, + INSTRUMENTATION_NAMES.CLOUDFLARE_AGENTS, + ), + CLOUDFLARE_WORKERS_CONTEXT, + ); + spans.set(event, span); + } catch (error) { + logInstrumentationError("start", error); + } + }, + asyncEnd: (event) => { + const span = spans.get(event); + if (!span) { + return; + } + spans.delete(event); + + try { + const status = ownValue(event.result, "status"); + if (status === "completed") { + span.log({ output: ownValue(event.result, "output") }); + } else { + const error = ownValue(event.result, "error"); + if (typeof error === "string") { + span.log({ error }); + } + } + } catch (error) { + logInstrumentationError("completion", error); + } finally { + safelyEndSpan(span); + } + }, + error: (event) => { + const span = spans.get(event); + if (!span) { + return; + } + spans.delete(event); + + try { + span.log({ error: event.error }); + } catch (error) { + logInstrumentationError("rejection", error); + } finally { + safelyEndSpan(span); + } + }, + }; + + channel.subscribe(handlers); +} + +function ownValue(value: unknown, key: PropertyKey): unknown { + if (!isObjectLike(value)) { + return undefined; + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + return descriptor && "value" in descriptor ? descriptor.value : undefined; +} + +function isObjectLike(value: unknown): value is object { + return ( + (typeof value === "object" && value !== null) || typeof value === "function" + ); +} + +function safelyEndSpan(span: Span): void { + try { + span.end(); + } catch (error) { + logInstrumentationError("span end", error); + } +} + +function logInstrumentationError(operation: string, error: unknown): void { + debugLogger.error( + `Failed to process Cloudflare Agents ${operation} instrumentation:`, + error, + ); +} diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-channels.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-channels.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-channels.ts diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.test.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts similarity index 94% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.test.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts index d20631a2d..860ab0b18 100644 --- a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.test.ts +++ b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.test.ts @@ -30,12 +30,11 @@ import { INSTRUMENTATION_NAMES, INTERNAL_SPAN_INSTRUMENTATION_NAME, } from "../../span-origin"; -import { CloudflareAIChatPlugin } from "./cloudflare-ai-chat-plugin"; +import { registerCloudflareAIChatInstrumentation } from "./cloudflare-ai-chat-consumer"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("CloudflareAIChatPlugin", () => { - let plugin: CloudflareAIChatPlugin; +describe("registerCloudflareAIChatInstrumentation", () => { let channels: Map>; beforeEach(() => { @@ -54,16 +53,14 @@ describe("CloudflareAIChatPlugin", () => { log: vi.fn(), })); mockInternalGetGlobalState.mockReturnValue(undefined); - plugin = new CloudflareAIChatPlugin(); }); afterEach(() => { - plugin.disable(); vi.clearAllMocks(); }); it("captures the full successful turn and binds queued work", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const turnHandlers = turnChannel().handlers(); const callback = vi.fn(async () => "callback-result"); const agent = { @@ -132,7 +129,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("correlates response errors and preserves partial output", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const turnHandlers = turnChannel().handlers(); const responseHandlers = responseChannel().handlers(); const agent = { messages: [], onChatResponse() {} }; @@ -169,7 +166,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("retains a settled turn until its queued response is observed", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [ @@ -225,7 +222,7 @@ describe("CloudflareAIChatPlugin", () => { it("drops retained turns that never produce a response", () => { vi.useFakeTimers(); try { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [], @@ -263,7 +260,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("preserves pre-turn input when a continuation reuses its output id", async () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [ @@ -324,7 +321,7 @@ describe("CloudflareAIChatPlugin", () => { }); it("deduplicates nested manual and automatic turn events", () => { - plugin.enable(); + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const agent = { messages: [], onChatResponse() {} }; const outer = { @@ -348,8 +345,8 @@ describe("CloudflareAIChatPlugin", () => { expect(span.end).toHaveBeenCalledTimes(1); }); - it("logs original errors and closes outstanding spans on disable", () => { - plugin.enable(); + it("logs original errors without closing unrelated outstanding spans", () => { + registerCloudflareAIChatInstrumentation(); const handlers = turnChannel().handlers(); const failure = new Error("turn failed"); const failedEvent = { @@ -370,8 +367,7 @@ describe("CloudflareAIChatPlugin", () => { } as any; handlers.start?.(pendingEvent, "start"); const pendingSpan = mockStartSpan.mock.results[1].value; - plugin.disable(); - expect(pendingSpan.end).toHaveBeenCalledTimes(1); + expect(pendingSpan.end).not.toHaveBeenCalled(); }); function turnChannel() { @@ -394,7 +390,6 @@ function createMockChannel() { hasSubscribers: false, start: { bindStore: vi.fn(), - unbindStore: vi.fn(), }, subscribe: vi.fn((handlers) => subscribed.push(handlers)), traceSync: vi.fn((callback, event) => { @@ -410,6 +405,5 @@ function createMockChannel() { throw error; } }), - unsubscribe: vi.fn(), }; } diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts similarity index 94% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts index d875824af..a75ad3ea6 100644 --- a/js/src/instrumentation/plugins/cloudflare-ai-chat-plugin.ts +++ b/js/src/instrumentation/providers/cloudflare-ai-chat-consumer.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { @@ -41,7 +40,7 @@ type TurnState = { const BLOCKED_KEYS = new Set(["__proto__", "constructor", "prototype"]); const RESPONSE_STATE_RETENTION_MS = 60_000; -export class CloudflareAIChatPlugin extends BasePlugin { +class CloudflareAIChatInstrumentationConsumer { private readonly activeStates = new Set(); private readonly activeTurns = new WeakMap< object, @@ -49,30 +48,18 @@ export class CloudflareAIChatPlugin extends BasePlugin { >(); private readonly eventStates = new WeakMap(); - protected onEnable(): void { + public register(): void { this.subscribeToResponseHook(); this.subscribeToTurnRunner(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - - for (const state of this.activeStates) { - this.cleanupState(state); - } - this.activeStates.clear(); - } - private subscribeToTurnRunner(): void { const tracingChannel = cloudflareAIChatChannels.runExclusiveChatTurn.tracingChannel() as IsoTracingChannel< ChannelMessage >; - const unbindCurrentSpanStore = this.bindCurrentSpanStore(tracingChannel); + this.bindCurrentSpanStore(tracingChannel); const handlers: IsoChannelHandlers> = { start: (event) => { this.ensureEventState(event); @@ -86,10 +73,6 @@ export class CloudflareAIChatPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToResponseHook(): void { @@ -143,12 +126,11 @@ export class CloudflareAIChatPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => tracingChannel.unsubscribe(handlers)); } private bindCurrentSpanStore( tracingChannel: IsoTracingChannel>, - ): (() => void) | undefined { + ): void { const globalState = _internalGetGlobalState(); const contextManager = globalState?.contextManager; const startChannel = tracingChannel.start; @@ -161,7 +143,7 @@ export class CloudflareAIChatPlugin extends BasePlugin { : undefined; if (!startChannel || !currentSpanStore || !contextManager) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -170,8 +152,6 @@ export class CloudflareAIChatPlugin extends BasePlugin { ? contextManager.wrapSpanForStore(state.span) : currentSpanStore.getStore(); }); - - return () => startChannel.unbindStore(currentSpanStore); } private ensureEventState( @@ -481,3 +461,13 @@ function sanitizeLoggedValue( } return Object.fromEntries(entries); } + +let cloudflareAIChatInstrumentationConsumer: + | CloudflareAIChatInstrumentationConsumer + | undefined; + +export function registerCloudflareAIChatInstrumentation(): void { + cloudflareAIChatInstrumentationConsumer ??= + new CloudflareAIChatInstrumentationConsumer(); + cloudflareAIChatInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/cloudflare-ai-chat-instrumentation.ts b/js/src/instrumentation/providers/cloudflare-ai-chat-instrumentation.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-ai-chat-instrumentation.ts rename to js/src/instrumentation/providers/cloudflare-ai-chat-instrumentation.ts diff --git a/js/src/instrumentation/plugins/cloudflare-think-channels.ts b/js/src/instrumentation/providers/cloudflare-think-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-think-channels.ts rename to js/src/instrumentation/providers/cloudflare-think-channels.ts diff --git a/js/src/instrumentation/plugins/cloudflare-think-context.ts b/js/src/instrumentation/providers/cloudflare-think-context.ts similarity index 100% rename from js/src/instrumentation/plugins/cloudflare-think-context.ts rename to js/src/instrumentation/providers/cloudflare-think-context.ts diff --git a/js/src/instrumentation/plugins/cloudflare-think-plugin.test.ts b/js/src/instrumentation/providers/cloudflare-think-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/cloudflare-think-plugin.test.ts rename to js/src/instrumentation/providers/cloudflare-think-instrumentation.test.ts index 240c33df0..f3e27919e 100644 --- a/js/src/instrumentation/plugins/cloudflare-think-plugin.test.ts +++ b/js/src/instrumentation/providers/cloudflare-think-instrumentation.test.ts @@ -27,7 +27,7 @@ describe("Cloudflare Think instrumentation", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "cloudflare-think-plugin.test.ts", + projectName: "cloudflare-think-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/cloudflare-think-plugin.ts b/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts similarity index 93% rename from js/src/instrumentation/plugins/cloudflare-think-plugin.ts rename to js/src/instrumentation/providers/cloudflare-think-instrumentation.ts index 8a29eb16c..66196becf 100644 --- a/js/src/instrumentation/plugins/cloudflare-think-plugin.ts +++ b/js/src/instrumentation/providers/cloudflare-think-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { @@ -25,7 +24,7 @@ import { finalizeAISDKChildTracing, patchAISDKStreamingResult, prepareAISDKAgentCallInput, -} from "./ai-sdk-plugin"; +} from "./ai-sdk-instrumentation"; import { cloudflareThinkChannels } from "./cloudflare-think-channels"; import { registerCloudflareThinkSpan, @@ -50,27 +49,15 @@ type AISDKStreamEvent = const THINK_STATE_ID = Symbol.for("braintrust.cloudflare-think.state-id"); -export class CloudflareThinkPlugin extends BasePlugin { +class CloudflareThinkInstrumentationConsumer { private readonly statesBySpanId = new Map(); - protected onEnable(): void { + public register(): void { this.subscribeToThinkRuns(); this.subscribeToAISDKStreamTextSync(); this.subscribeToAISDKStreamTextAsync(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - - for (const state of this.statesBySpanId.values()) { - this.finishState(state); - } - this.statesBySpanId.clear(); - } - private subscribeToThinkRuns(): void { const channel = cloudflareThinkChannels.runInferenceLoop.tracingChannel(); const states = new WeakMap(); @@ -137,9 +124,6 @@ export class CloudflareThinkPlugin extends BasePlugin { ? contextManager.wrapSpanForStore(runState.span) : currentSpanStore.getStore(); }); - this.unsubscribers.push(() => - channel.start?.unbindStore(currentSpanStore), - ); } const handlers: IsoChannelHandlers< @@ -166,7 +150,6 @@ export class CloudflareThinkPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } private subscribeToAISDKStreamTextSync(): void { @@ -186,7 +169,6 @@ export class CloudflareThinkPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } private subscribeToAISDKStreamTextAsync(): void { @@ -206,7 +188,6 @@ export class CloudflareThinkPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } private startAISDKStream(event: AISDKStreamEvent): void { @@ -382,3 +363,13 @@ function extractThinkTaskInput(input: unknown): unknown { } return input; } + +let cloudflareThinkInstrumentationConsumer: + | CloudflareThinkInstrumentationConsumer + | undefined; + +export function registerCloudflareThinkInstrumentation(): void { + cloudflareThinkInstrumentationConsumer ??= + new CloudflareThinkInstrumentationConsumer(); + cloudflareThinkInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/cohere-channels.ts b/js/src/instrumentation/providers/cohere-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cohere-channels.ts rename to js/src/instrumentation/providers/cohere-channels.ts diff --git a/js/src/instrumentation/plugins/cohere-plugin.test.ts b/js/src/instrumentation/providers/cohere-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/cohere-plugin.test.ts rename to js/src/instrumentation/providers/cohere-instrumentation.test.ts index c2ea4fbf1..d7bd3603c 100644 --- a/js/src/instrumentation/plugins/cohere-plugin.test.ts +++ b/js/src/instrumentation/providers/cohere-instrumentation.test.ts @@ -3,7 +3,7 @@ import { aggregateCohereChatStreamChunks, extractCohereResponseMetadata, parseCohereMetricsFromUsage, -} from "./cohere-plugin"; +} from "./cohere-instrumentation"; describe("parseCohereMetricsFromUsage", () => { it("parses usage and meta token counters", () => { diff --git a/js/src/instrumentation/plugins/cohere-plugin.ts b/js/src/instrumentation/providers/cohere-instrumentation.ts similarity index 89% rename from js/src/instrumentation/plugins/cohere-plugin.ts rename to js/src/instrumentation/providers/cohere-instrumentation.ts index afaea628c..14e21179c 100644 --- a/js/src/instrumentation/plugins/cohere-plugin.ts +++ b/js/src/instrumentation/providers/cohere-instrumentation.ts @@ -1,8 +1,6 @@ -import { BasePlugin } from "../core"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { processInputAttachments } from "../../wrappers/attachment-utils"; @@ -16,79 +14,67 @@ import type { CohereUsageLike, } from "../../vendor-sdk-types/cohere"; -export class CoherePlugin extends BasePlugin { - protected onEnable(): void { +class CohereInstrumentationConsumer { + public register(): void { this.subscribeToCohereChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToCohereChannels(): void { - this.unsubscribers.push( - traceStreamingChannel(cohereChannels.chat, { - name: "cohere.chat", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - extractOutput: (result) => extractCohereChatOutput(result), - extractMetadata: (result) => extractCohereResponseMetadata(result), - extractMetrics: (result, startTime) => { - const metrics = parseCohereMetricsFromUsage(result); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - }), - ); + traceStreamingChannel(cohereChannels.chat, { + name: "cohere.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + extractOutput: (result) => extractCohereChatOutput(result), + extractMetadata: (result) => extractCohereResponseMetadata(result), + extractMetrics: (result, startTime) => { + const metrics = parseCohereMetricsFromUsage(result); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + }); - this.unsubscribers.push( - traceStreamingChannel(cohereChannels.chatStream, { - name: "cohere.chatStream", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - extractOutput: () => undefined, - extractMetadata: () => undefined, - extractMetrics: () => ({}), - aggregateChunks: aggregateCohereChatStreamChunks, - }), - ); + traceStreamingChannel(cohereChannels.chatStream, { + name: "cohere.chatStream", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + extractOutput: () => undefined, + extractMetadata: () => undefined, + extractMetrics: () => ({}), + aggregateChunks: aggregateCohereChatStreamChunks, + }); - this.unsubscribers.push( - traceAsyncChannel(cohereChannels.embed, { - name: "cohere.embed", - type: SpanTypeAttribute.LLM, - extractInput: extractEmbedInputWithMetadata, - extractOutput: extractCohereEmbeddingOutput, - extractMetadata: (result) => extractCohereResponseMetadata(result), - extractMetrics: (result) => parseCohereMetricsFromUsage(result), - }), - ); + traceAsyncChannel(cohereChannels.embed, { + name: "cohere.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractEmbedInputWithMetadata, + extractOutput: extractCohereEmbeddingOutput, + extractMetadata: (result) => extractCohereResponseMetadata(result), + extractMetrics: (result) => parseCohereMetricsFromUsage(result), + }); - this.unsubscribers.push( - traceAsyncChannel(cohereChannels.rerank, { - name: "cohere.rerank", - type: SpanTypeAttribute.LLM, - extractInput: extractRerankInputWithMetadata, - extractOutput: (result) => { - if (!isObject(result) || !Array.isArray(result.results)) { - return undefined; - } - - return result.results.slice(0, 100).map((item) => ({ - index: isObject(item) ? item.index : undefined, - relevance_score: isObject(item) - ? ((typeof item.relevanceScore === "number" - ? item.relevanceScore - : item.relevance_score) ?? null) - : null, - })); - }, - extractMetadata: (result) => extractCohereResponseMetadata(result), - extractMetrics: (result) => parseCohereMetricsFromUsage(result), - }), - ); + traceAsyncChannel(cohereChannels.rerank, { + name: "cohere.rerank", + type: SpanTypeAttribute.LLM, + extractInput: extractRerankInputWithMetadata, + extractOutput: (result) => { + if (!isObject(result) || !Array.isArray(result.results)) { + return undefined; + } + + return result.results.slice(0, 100).map((item) => ({ + index: isObject(item) ? item.index : undefined, + relevance_score: isObject(item) + ? ((typeof item.relevanceScore === "number" + ? item.relevanceScore + : item.relevance_score) ?? null) + : null, + })); + }, + extractMetadata: (result) => extractCohereResponseMetadata(result), + extractMetrics: (result) => parseCohereMetricsFromUsage(result), + }); } } @@ -949,3 +935,10 @@ export function aggregateCohereChatStreamChunks( output, }; } + +let cohereInstrumentationConsumer: CohereInstrumentationConsumer | undefined; + +export function registerCohereInstrumentation(): void { + cohereInstrumentationConsumer ??= new CohereInstrumentationConsumer(); + cohereInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/cursor-sdk-channels.ts b/js/src/instrumentation/providers/cursor-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/cursor-sdk-channels.ts rename to js/src/instrumentation/providers/cursor-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/cursor-sdk-plugin.test.ts b/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts similarity index 94% rename from js/src/instrumentation/plugins/cursor-sdk-plugin.test.ts rename to js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts index b22052c8c..c30169c35 100644 --- a/js/src/instrumentation/plugins/cursor-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/cursor-sdk-instrumentation.test.ts @@ -15,11 +15,11 @@ vi.mock("../../logger", () => ({ })); import iso from "../../isomorph"; -import { CursorSDKPlugin } from "./cursor-sdk-plugin"; +import { registerCursorSDKInstrumentation } from "./cursor-sdk-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("CursorSDKPlugin", () => { +describe("registerCursorSDKInstrumentation", () => { let handlersByName: Map; let spans: Array<{ end: ReturnType; @@ -34,7 +34,6 @@ describe("CursorSDKPlugin", () => { mockNewTracingChannel.mockImplementation((name: string) => ({ subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), tracePromise: vi.fn((fn) => fn()), - unsubscribe: vi.fn(), })); mockStartSpan.mockImplementation((args: any) => { const span = { @@ -56,9 +55,7 @@ describe("CursorSDKPlugin", () => { }); it("subscribes to Cursor SDK channels", () => { - const plugin = new CursorSDKPlugin(); - - plugin.enable(); + registerCursorSDKInstrumentation(); expect(handlersByName.has("orchestrion:@cursor/sdk:Agent.create")).toBe( true, @@ -73,8 +70,7 @@ describe("CursorSDKPlugin", () => { }); it("patches agents returned by Agent.create and traces send/wait", async () => { - const plugin = new CursorSDKPlugin(); - plugin.enable(); + registerCursorSDKInstrumentation(); const createHandlers = handlersByName.get( "orchestrion:@cursor/sdk:Agent.create", @@ -131,8 +127,7 @@ describe("CursorSDKPlugin", () => { }); it("captures stream tool calls and usage", async () => { - const plugin = new CursorSDKPlugin(); - plugin.enable(); + registerCursorSDKInstrumentation(); const sendHandlers = handlersByName.get( "orchestrion:@cursor/sdk:agent.send", @@ -214,8 +209,7 @@ describe("CursorSDKPlugin", () => { }); it("traces Agent.prompt without a nested send span", () => { - const plugin = new CursorSDKPlugin(); - plugin.enable(); + registerCursorSDKInstrumentation(); const promptHandlers = handlersByName.get( "orchestrion:@cursor/sdk:Agent.prompt", diff --git a/js/src/instrumentation/plugins/cursor-sdk-plugin.ts b/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts similarity index 98% rename from js/src/instrumentation/plugins/cursor-sdk-plugin.ts rename to js/src/instrumentation/providers/cursor-sdk-instrumentation.ts index f01db5987..41118be17 100644 --- a/js/src/instrumentation/plugins/cursor-sdk-plugin.ts +++ b/js/src/instrumentation/providers/cursor-sdk-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; @@ -64,23 +64,15 @@ type PromptState = { startTime: number; }; -export class CursorSDKPlugin extends BasePlugin { +class CursorSDKInstrumentationConsumer { private promptDepth = 0; - protected onEnable(): void { + public register(): void { this.subscribeToAgentFactories(); this.subscribeToPrompt(); this.subscribeToSend(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - this.promptDepth = 0; - } - private subscribeToAgentFactories(): void { this.subscribeToAgentFactory(cursorSDKChannels.create); this.subscribeToAgentFactory(cursorSDKChannels.resume); @@ -98,9 +90,6 @@ export class CursorSDKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private subscribeToPrompt(): void { @@ -171,9 +160,6 @@ export class CursorSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } private subscribeToSend(): void { @@ -267,9 +253,6 @@ export class CursorSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } } @@ -1185,3 +1168,12 @@ function cleanMetrics(metrics: Record): Record { } return cleaned; } + +let cursorSDKInstrumentationConsumer: + | CursorSDKInstrumentationConsumer + | undefined; + +export function registerCursorSDKInstrumentation(): void { + cursorSDKInstrumentationConsumer ??= new CursorSDKInstrumentationConsumer(); + cursorSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/providers/eve-instrumentation.test.ts b/js/src/instrumentation/providers/eve-instrumentation.test.ts new file mode 100644 index 000000000..9e1a6bc28 --- /dev/null +++ b/js/src/instrumentation/providers/eve-instrumentation.test.ts @@ -0,0 +1,1344 @@ +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import * as braintrustExports from "../../exports"; +import { configureNode } from "../../node/config"; +import { Attachment, _exportsForTestingOnly, initLogger } from "../../logger"; +import * as instrumentationExports from "../index"; +import { braintrustEveInstrumentation } from "./eve-instrumentation"; +import type { + EveInstrumentationAttemptScope, + EveInstrumentationHandlerContext, + EveInstrumentationModelCallCompletedEvent, + EveJsonValue, +} from "../../vendor-sdk-types/eve"; + +const ATTEMPT_SCOPE: EveInstrumentationAttemptScope = { + attemptId: "attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-root", + stepIndex: 0, + turnId: "turn-0", +}; + +function providerContext( + initial?: EveJsonValue, +): EveInstrumentationHandlerContext { + let value = initial; + return { + state: { + get: () => value, + set: (next) => { + value = next; + }, + }, + }; +} + +try { + configureNode(); +} catch { + // Best-effort initialization for test environments. +} + +describe("braintrustEveInstrumentation", () => { + let backgroundLogger: ReturnType< + typeof _exportsForTestingOnly.useTestBackgroundLogger + >; + + beforeAll(async () => { + await _exportsForTestingOnly.simulateLoginForTests(); + }); + + beforeEach(() => { + backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); + initLogger({ + projectName: "eve-instrumentation.test.ts", + projectId: "test-project-id", + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + _exportsForTestingOnly.clearTestBackgroundLogger(); + }); + + it("returns an Eve 0.34+ instrumentation provider", () => { + const setup = vi.fn(); + const provider = braintrustEveInstrumentation({ setup }); + + expect(provider.capture).toBe("content"); + expect(provider.setup).toBe(setup); + expect(typeof provider.flush).toBe("function"); + expect(Object.keys(provider.events).sort()).toEqual([ + "action.completed", + "action.failed", + "action.started", + "model.call.completed", + "model.call.failed", + "model.call.started", + "step.attempt.completed", + "step.attempt.failed", + "turn.cancelled", + "turn.completed", + "turn.failed", + "turn.started", + ]); + }); + + it("exports only the Eve instrumentation provider API", () => { + expect(braintrustExports.braintrustEveInstrumentation).toBe( + braintrustEveInstrumentation, + ); + expect(instrumentationExports.braintrustEveInstrumentation).toBe( + braintrustEveInstrumentation, + ); + expect(braintrustExports).not.toHaveProperty("braintrustEveHook"); + expect(instrumentationExports).not.toHaveProperty("braintrustEveHook"); + }); + + it("records a turn with LLM and action spans", async () => { + const provider = braintrustEveInstrumentation({ + metadata: { + scenario: "eve-instrumentation-unit", + testRunId: "test-run-flat-tree", + }, + }); + const turnContext = providerContext(); + const firstModelContext = providerContext(); + const actionContext = providerContext(); + const secondModelContext = providerContext(); + const scope = ATTEMPT_SCOPE; + + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + turnContext, + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + instructions: "You are a research agent.", + messages: [{ content: "Search then read", role: "user" }], + }, + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope, + type: "model.call.started", + }, + firstModelContext, + ); + await provider.events["model.call.completed"]( + { + content: [ + { text: "I should search first.", type: "reasoning" }, + { + callId: "call-search", + input: { query: "Eve instrumentation" }, + toolName: "search", + type: "tool-call", + }, + ], + finishReason: "tool-calls", + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.completed", + usage: { + inputTokenDetails: { + cacheReadTokens: 3, + cacheWriteTokens: 2, + }, + inputTokens: 10, + outputTokens: 5, + }, + }, + firstModelContext, + ); + await provider.events["action.started"]( + { + callId: "call-search", + idempotencyKey: "action:session-root:turn-0:call-search", + input: { query: "Eve instrumentation" }, + name: "search", + scope, + type: "action.started", + }, + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-search", + output: { + output: { hits: ["eve.dev/docs"] }, + type: "result", + }, + scope, + type: "action.completed", + }, + actionContext, + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-1:0", + input: { + instructions: "You are a research agent.", + messages: [ + { content: "Search then read", role: "user" }, + { + content: [ + { + input: { query: "Eve instrumentation" }, + toolCallId: "call-search", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ], + }, + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope, + type: "model.call.started", + }, + secondModelContext, + ); + await provider.events["model.call.completed"]( + { + content: [ + { + text: "Here is the Eve instrumentation guide.", + type: "text", + }, + ], + finishReason: "stop", + idempotencyKey: "model:attempt-1:0", + scope, + type: "model.call.completed", + usage: { + inputTokens: 20, + outputTokens: 8, + }, + }, + secondModelContext, + ); + await provider.events["turn.completed"]( + { + idempotencyKey: "turn:session-root:turn-0", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.completed", + }, + turnContext, + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const root = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + const steps = spans.filter( + (span) => span.span_attributes?.name === "eve.step", + ); + const tool = spans.find((span) => span.span_attributes?.name === "search"); + + expect(spans.map((span) => span.span_attributes?.name)).toEqual([ + "eve.turn", + "eve.step", + "search", + "eve.step", + ]); + expect(root).toMatchObject({ + input: [{ content: "Search then read", role: "user" }], + metadata: { + "eve.session_id": "session-root", + scenario: "eve-instrumentation-unit", + testRunId: "test-run-flat-tree", + }, + output: "Here is the Eve instrumentation guide.", + metrics: { + completion_tokens: 13, + prompt_cache_creation_tokens: 2, + prompt_cached_tokens: 3, + prompt_tokens: 30, + tokens: 43, + }, + span_attributes: { + name: "eve.turn", + type: "task", + }, + span_parents: [], + }); + expect(root?.root_span_id).not.toBe(root?.span_id); + expect(root?.metadata).not.toHaveProperty("model"); + expect(root?.metadata).not.toHaveProperty("provider"); + + expect(steps).toHaveLength(2); + for (const step of steps) { + expect(step).toMatchObject({ + metadata: { + "eve.session_id": "session-root", + model: "qwen/qwen3-30b-a3b", + provider: "openrouter", + scenario: "eve-instrumentation-unit", + testRunId: "test-run-flat-tree", + }, + span_attributes: { + name: "eve.step", + type: "llm", + }, + span_parents: [root?.span_id], + }); + } + expect(steps[0]?.metadata).not.toHaveProperty("tools"); + expect(steps[0]?.output).toMatchObject([ + { + finish_reason: "tool_calls", + message: { + content: null, + reasoning: [{ content: "I should search first." }], + tool_calls: [ + { + function: { + arguments: JSON.stringify({ + query: "Eve instrumentation", + }), + name: "search", + }, + id: "call-search", + type: "function", + }, + ], + }, + }, + ]); + expect(steps[1]?.output).toMatchObject([ + { + finish_reason: "stop", + message: { + content: "Here is the Eve instrumentation guide.", + role: "assistant", + }, + }, + ]); + expect(tool).toMatchObject({ + input: { query: "Eve instrumentation" }, + metadata: { + "eve.session_id": "session-root", + scenario: "eve-instrumentation-unit", + testRunId: "test-run-flat-tree", + }, + output: { hits: ["eve.dev/docs"] }, + span_attributes: { + name: "search", + type: "tool", + }, + span_parents: [root?.span_id], + }); + expect(tool?.metadata).not.toHaveProperty("model"); + expect(tool?.metadata).not.toHaveProperty("provider"); + }); + + it("attaches a subagent turn beneath its action span", async () => { + const provider = braintrustEveInstrumentation(); + const parentTurnContext = providerContext(); + const actionContext = providerContext(); + const childTurnContext = providerContext(); + const childModelContext = providerContext(); + const parentScope = ATTEMPT_SCOPE; + + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + parentTurnContext, + ); + await provider.events["action.started"]( + { + callId: "call-researcher", + idempotencyKey: "action:session-root:turn-0:call-researcher", + input: { message: "Research Eve" }, + name: "researcher", + scope: parentScope, + type: "action.started", + }, + actionContext, + ); + await Promise.all([ + provider.events["turn.started"]( + { + idempotencyKey: "turn:session-child:turn-0", + parentLineage: { + callId: "call-researcher", + sessionId: "session-root", + turnId: "turn-0", + }, + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-child", + turnId: "turn-0", + type: "turn.started", + }, + childTurnContext, + ), + provider.events["model.call.started"]( + { + idempotencyKey: "model:child-attempt-0:0", + model: { + modelId: "qwen/qwen3-30b-a3b", + provider: "openrouter.chat", + }, + scope: { + attemptId: "child-attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "turn-0", + }, + type: "model.call.started", + }, + childModelContext, + ), + ]); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const parent = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-root", + ); + const action = spans.find( + (span) => span.span_attributes?.name === "researcher", + ); + const child = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-child", + ); + const childModel = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + + expect(action?.span_parents).toEqual([parent?.span_id]); + expect(child?.span_parents).toEqual([action?.span_id]); + expect(child?.root_span_id).toBe(parent?.root_span_id); + expect(childModel?.span_parents).toEqual([child?.span_id]); + expect(childModel?.root_span_id).toBe(parent?.root_span_id); + }); + + it("keeps a stable root across nested subagents with different turn ids", async () => { + const provider = braintrustEveInstrumentation(); + const rootContext = providerContext(); + const rootScope = { ...ATTEMPT_SCOPE, turnId: "root-turn" }; + const childScope: EveInstrumentationAttemptScope = { + attemptId: "child-attempt", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "child-turn", + }; + + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:root-turn", + rootSessionId: "session-root", + sequence: 4, + sessionId: "session-root", + turnId: "root-turn", + type: "turn.started", + }, + rootContext, + ); + await provider.events["action.started"]( + { + callId: "call-child", + idempotencyKey: "action:session-root:root-turn:call-child", + name: "child", + scope: rootScope, + type: "action.started", + }, + providerContext(), + ); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-child:child-turn", + parentLineage: { + callId: "call-child", + sessionId: "session-root", + turnId: "root-turn", + }, + rootSessionId: "session-root", + sequence: 2, + sessionId: "session-child", + turnId: "child-turn", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["action.started"]( + { + callId: "call-grandchild", + idempotencyKey: "action:session-child:child-turn:call-grandchild", + name: "grandchild", + scope: childScope, + type: "action.started", + }, + providerContext(), + ); + const rootReference = rootContext.state.get(); + if ( + typeof rootReference !== "object" || + rootReference === null || + Array.isArray(rootReference) || + typeof rootReference.rootSpanId !== "string" + ) { + throw new Error("Expected the root turn trace context to be persisted"); + } + const resumedProvider = braintrustEveInstrumentation(); + await resumedProvider.events["turn.started"]( + { + idempotencyKey: "turn:session-grandchild:grandchild-turn", + parentLineage: { + callId: "call-grandchild", + sessionId: "session-child", + turnId: "child-turn", + }, + parentTraceContext: { + spanId: "1111111111111111", + traceFlags: 1, + traceId: rootReference.rootSpanId, + }, + rootSessionId: "session-root", + sequence: 7, + sessionId: "session-grandchild", + turnId: "grandchild-turn", + type: "turn.started", + }, + providerContext(), + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const turns = spans.filter( + (span) => span.span_attributes?.name === "eve.turn", + ); + const root = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-root", + ); + const child = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-child", + ); + const grandchild = turns.find( + (turn) => turn.metadata?.["eve.session_id"] === "session-grandchild", + ); + const childAction = spans.find( + (span) => span.span_attributes?.name === "child", + ); + const grandchildAction = spans.find( + (span) => span.span_attributes?.name === "grandchild", + ); + + expect(child?.span_parents).toEqual([childAction?.span_id]); + expect(grandchild?.span_parents).toEqual([grandchildAction?.span_id]); + expect(turns.map((turn) => turn.root_span_id)).toEqual([ + root?.root_span_id, + root?.root_span_id, + root?.root_span_id, + ]); + }); + + it("creates a separate trace for each top-level turn in one session", async () => { + const provider = braintrustEveInstrumentation(); + + await provider.events["turn.started"]( + { + idempotencyKey: "turn:shared-session:turn-0", + rootSessionId: "shared-session", + sequence: 0, + sessionId: "shared-session", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:shared-session:turn-1", + rootSessionId: "shared-session", + sequence: 1, + sessionId: "shared-session", + turnId: "turn-1", + type: "turn.started", + }, + providerContext(), + ); + + const turns = (await backgroundLogger.drain()) as Array< + Record + >; + expect(turns).toHaveLength(2); + expect(turns.map((turn) => turn.metadata?.["eve.session_id"])).toEqual([ + "shared-session", + "shared-session", + ]); + expect(new Set(turns.map((turn) => turn.root_span_id)).size).toBe(2); + }); + + it("preserves a durable subagent turn across provider instances", async () => { + const firstProvider = braintrustEveInstrumentation(); + const parentTurnContext = providerContext(); + const childTurnContext = providerContext(); + + await firstProvider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + parentTurnContext, + ); + await firstProvider.events["action.started"]( + { + callId: "call-researcher", + idempotencyKey: "action:session-root:turn-0:call-researcher", + name: "researcher", + scope: ATTEMPT_SCOPE, + type: "action.started", + }, + providerContext(), + ); + await firstProvider.events["turn.started"]( + { + idempotencyKey: "turn:session-child:turn-0", + parentLineage: { + callId: "call-researcher", + sessionId: "session-root", + turnId: "turn-0", + }, + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-child", + turnId: "turn-0", + type: "turn.started", + }, + childTurnContext, + ); + + const resumedProvider = braintrustEveInstrumentation(); + const childScope: EveInstrumentationAttemptScope = { + attemptId: "child-attempt-0", + attemptIndex: 0, + rootSessionId: "session-root", + sessionId: "session-child", + stepIndex: 0, + turnId: "turn-0", + }; + const modelContext = providerContext(); + await resumedProvider.events["model.call.started"]( + { + idempotencyKey: "model:child-attempt-0:0", + input: { + messages: [{ content: "Continue durably", role: "user" }], + }, + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: childScope, + type: "model.call.started", + }, + modelContext, + ); + await resumedProvider.events["model.call.completed"]( + { + content: [{ text: "Durable answer", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:child-attempt-0:0", + scope: childScope, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 2 }, + }, + modelContext, + ); + await resumedProvider.events["turn.completed"]( + { + idempotencyKey: "turn:session-child:turn-0", + sessionId: "session-child", + turnId: "turn-0", + type: "turn.completed", + }, + providerContext(childTurnContext.state.get()), + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const parent = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-root", + ); + const action = spans.find( + (span) => span.span_attributes?.name === "researcher", + ); + const child = spans.find( + (span) => + span.span_attributes?.name === "eve.turn" && + span.metadata?.["eve.session_id"] === "session-child", + ); + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(child).toMatchObject({ + input: [{ content: "Continue durably", role: "user" }], + metrics: { + completion_tokens: 2, + prompt_tokens: 3, + tokens: 5, + }, + output: "Durable answer", + span_parents: [action?.span_id], + }); + expect(child?.root_span_id).toBe(parent?.root_span_id); + expect(step?.span_parents).toEqual([child?.span_id]); + expect(step?.root_span_id).toBe(parent?.root_span_id); + }); + + it("resumes spans and turn metrics from Eve operation state", async () => { + const firstProvider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + const scope = ATTEMPT_SCOPE; + + await firstProvider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + turnContext, + ); + await firstProvider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + messages: [{ content: "Resume this call", role: "user" }], + }, + model: { + modelId: "gpt-5.4-mini", + provider: "openai.responses", + }, + scope, + type: "model.call.started", + }, + modelContext, + ); + + const persisted = modelContext.state.get(); + expect(persisted).toMatchObject({ + exported: expect.any(String), + rootSpanId: expect.any(String), + spanId: expect.any(String), + }); + + const resumedProvider = braintrustEveInstrumentation(); + await resumedProvider.events["model.call.completed"]( + { + content: [{ text: "Resumed output", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.completed", + usage: { inputTokens: 4, outputTokens: 2 }, + }, + providerContext(persisted), + ); + const secondModelContext = providerContext(); + await resumedProvider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:1", + input: { + messages: [{ content: "Continue this call", role: "user" }], + }, + model: { + modelId: "gpt-5.4-mini", + provider: "openai.responses", + }, + scope: { ...scope, stepIndex: 1 }, + type: "model.call.started", + }, + secondModelContext, + ); + const secondPersisted = secondModelContext.state.get(); + expect(secondPersisted).toMatchObject({ + turnMetricContributions: { + "model:attempt-0:0": { + completion_tokens: 2, + prompt_tokens: 4, + tokens: 6, + }, + }, + }); + const finalProvider = braintrustEveInstrumentation(); + await finalProvider.events["model.call.completed"]( + { + content: [{ text: "Final resumed output", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:attempt-0:1", + scope: { ...scope, stepIndex: 1 }, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 1 }, + }, + providerContext(secondPersisted), + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const steps = spans.filter( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(steps).toHaveLength(2); + expect(steps[0]).toMatchObject({ + input: [{ content: "Resume this call", role: "user" }], + metrics: { + completion_tokens: 2, + prompt_tokens: 4, + tokens: 6, + }, + output: [ + { + finish_reason: "stop", + message: { + content: "Resumed output", + role: "assistant", + }, + }, + ], + }); + const turn = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + expect(turn?.metrics).toMatchObject({ + completion_tokens: 3, + prompt_tokens: 7, + tokens: 10, + }); + }); + + it("does not double-count replayed model completions", async () => { + const provider = braintrustEveInstrumentation(); + const modelContext = providerContext(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:replay-session:turn-0", + rootSessionId: "replay-session", + sequence: 0, + sessionId: "replay-session", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + const scope = { + ...ATTEMPT_SCOPE, + attemptId: "replay-attempt", + rootSessionId: "replay-session", + sessionId: "replay-session", + }; + await provider.events["model.call.started"]( + { + idempotencyKey: "model:replay-attempt:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope, + type: "model.call.started", + }, + modelContext, + ); + const completed: EveInstrumentationModelCallCompletedEvent = { + content: [{ text: "Done", type: "text" }], + finishReason: "stop", + idempotencyKey: "model:replay-attempt:0", + scope, + type: "model.call.completed", + usage: { inputTokens: 3, outputTokens: 2 }, + }; + await provider.events["model.call.completed"](completed, modelContext); + await provider.events["model.call.completed"](completed, modelContext); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const turn = spans.find( + (span) => span.span_attributes?.name === "eve.turn", + ); + expect(turn?.metrics).toMatchObject({ + completion_tokens: 2, + prompt_tokens: 3, + tokens: 5, + }); + }); + + it("records model, action, and turn failures as errors", async () => { + const provider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + const actionContext = providerContext(); + const scope = ATTEMPT_SCOPE; + + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + turnContext, + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope, + type: "model.call.started", + }, + modelContext, + ); + await provider.events["model.call.failed"]( + { + error: new Error("model exploded"), + idempotencyKey: "model:attempt-0:0", + scope, + type: "model.call.failed", + }, + modelContext, + ); + await provider.events["action.started"]( + { + callId: "call-failing", + idempotencyKey: "action:session-root:turn-0:call-failing", + name: "failing-tool", + scope, + type: "action.started", + }, + actionContext, + ); + await provider.events["action.failed"]( + { + error: new Error("tool exploded"), + errorCode: "TOOL_FAILED", + idempotencyKey: "action:session-root:turn-0:call-failing", + outcome: "failed", + scope, + type: "action.failed", + }, + actionContext, + ); + await provider.events["turn.failed"]( + { + error: new Error("turn exploded"), + idempotencyKey: "turn:session-root:turn-0", + sessionId: "session-root", + turnId: "turn-0", + type: "turn.failed", + }, + turnContext, + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + expect( + spans.find((span) => span.span_attributes?.name === "eve.step")?.error, + ).toContain("model exploded"); + expect( + spans.find((span) => span.span_attributes?.name === "failing-tool") + ?.error, + ).toContain("tool exploded"); + expect( + spans.find((span) => span.span_attributes?.name === "eve.turn")?.error, + ).toContain("turn exploded"); + }); + + it("projects model input without provider-private fields", async () => { + const provider = braintrustEveInstrumentation(); + const turnContext = providerContext(); + const modelContext = providerContext(); + + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + turnContext, + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + instructions: { + content: "Object system instruction", + providerOptions: { encrypted: "instruction-secret" }, + role: "system", + }, + messages: [ + { + content: "Hello", + providerOptions: { reasoning_details: "message-secret" }, + role: "user", + }, + { + content: [ + { + providerOptions: { encrypted: "part-secret" }, + text: "Thinking", + type: "reasoning", + }, + { + input: { query: "Eve" }, + providerOptions: { encrypted: "tool-secret" }, + toolCallId: "call-0", + toolName: "search", + type: "tool-call", + }, + { + providerOptions: { encrypted: "custom-secret" }, + type: "custom", + }, + ], + providerOptions: { reasoning_details: "assistant-secret" }, + role: "assistant", + }, + ], + }, + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", + }, + modelContext, + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(step?.input).toEqual([ + { content: "Object system instruction", role: "system" }, + { content: "Hello", role: "user" }, + { + content: [ + { text: "Thinking", type: "reasoning" }, + { + input: { query: "Eve" }, + toolCallId: "call-0", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ]); + expect(JSON.stringify(step?.input)).not.toContain("providerOptions"); + expect(JSON.stringify(step?.input)).not.toContain("secret"); + }); + + it("converts inline model and tool-result media to attachments", async () => { + const provider = braintrustEveInstrumentation(); + const actionContext = providerContext(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + input: { + messages: [ + { + content: [ + { + data: "AQID", + filename: "tiny.png", + mediaType: "image/png", + type: "file", + }, + { + image: "DQ4P", + mediaType: "image/png", + type: "image", + }, + { + data: "not valid base64!", + filename: "invalid.png", + mediaType: "image/png", + type: "file", + }, + ], + role: "user", + }, + { + content: [ + { + output: { + type: "content", + value: [ + { + data: "BAUG", + mediaType: "image/png", + type: "image-data", + }, + ], + }, + toolCallId: "call-image", + toolName: "render", + type: "tool-result", + }, + ], + role: "tool", + }, + ], + }, + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", + }, + providerContext(), + ); + await provider.events["action.started"]( + { + callId: "call-render", + idempotencyKey: "action:session-root:turn-0:call-render", + input: { + data: "data:image/png;base64,BwgJ", + mediaType: "image/png", + type: "file", + }, + name: "render", + scope: ATTEMPT_SCOPE, + type: "action.started", + }, + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-render", + output: { + output: { + type: "content", + value: [ + { + data: new Uint8Array([10, 11, 12]), + mediaType: "image/png", + type: "image-data", + }, + ], + }, + type: "result", + }, + scope: ATTEMPT_SCOPE, + type: "action.completed", + }, + actionContext, + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + const inputAttachment = step?.input?.[0]?.content?.[0]?.data; + const imageAttachment = step?.input?.[0]?.content?.[1]?.image; + const unconvertedFile = step?.input?.[0]?.content?.[2]?.data; + const outputAttachment = + step?.input?.[1]?.content?.[0]?.output?.value?.[0]?.data; + const action = spans.find( + (span) => span.span_attributes?.name === "render", + ); + const actionInputAttachment = action?.input?.data; + const actionOutputAttachment = action?.output?.value?.[0]?.data; + expect(inputAttachment).toBeInstanceOf(Attachment); + expect(inputAttachment.reference).toMatchObject({ + content_type: "image/png", + filename: "tiny.png", + type: "braintrust_attachment", + }); + expect(imageAttachment).toBeInstanceOf(Attachment); + expect(imageAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(unconvertedFile).toBe("not valid base64!"); + expect(outputAttachment).toBeInstanceOf(Attachment); + expect(outputAttachment.reference).toMatchObject({ + content_type: "image/png", + filename: "attachment.png", + type: "braintrust_attachment", + }); + expect(actionInputAttachment).toBeInstanceOf(Attachment); + expect(actionInputAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(actionOutputAttachment).toBeInstanceOf(Attachment); + expect(actionOutputAttachment.reference).toMatchObject({ + content_type: "image/png", + type: "braintrust_attachment", + }); + expect(JSON.stringify(step?.input)).not.toContain("AQID"); + expect(JSON.stringify(step?.input)).not.toContain("BAUG"); + }); + + it("preserves arbitrary action outputs with type fields", async () => { + const provider = braintrustEveInstrumentation(); + const actionContext = providerContext(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["action.started"]( + { + callId: "call-text", + idempotencyKey: "action:session-root:turn-0:call-text", + name: "text-result", + scope: ATTEMPT_SCOPE, + type: "action.started", + }, + actionContext, + ); + await provider.events["action.completed"]( + { + idempotencyKey: "action:session-root:turn-0:call-text", + output: { + output: { text: "preserve me", type: "text" }, + type: "result", + }, + scope: ATTEMPT_SCOPE, + type: "action.completed", + }, + actionContext, + ); + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + expect( + spans.find((span) => span.span_attributes?.name === "text-result") + ?.output, + ).toEqual({ text: "preserve me", type: "text" }); + }); + + it.each([ + ["step.attempt.completed", undefined], + ["step.attempt.failed", new Error("attempt exploded")], + ] as const)("closes an open model span on %s", async (type, error) => { + const provider = braintrustEveInstrumentation(); + await provider.events["turn.started"]( + { + idempotencyKey: "turn:session-root:turn-0", + rootSessionId: "session-root", + sequence: 0, + sessionId: "session-root", + turnId: "turn-0", + type: "turn.started", + }, + providerContext(), + ); + await provider.events["model.call.started"]( + { + idempotencyKey: "model:attempt-0:0", + model: { modelId: "gpt-5.4-mini", provider: "openai.responses" }, + scope: ATTEMPT_SCOPE, + type: "model.call.started", + }, + providerContext(), + ); + + if (type === "step.attempt.completed") { + await provider.events[type]( + { + idempotencyKey: "step:attempt-0", + scope: ATTEMPT_SCOPE, + type, + }, + providerContext(), + ); + } else { + await provider.events[type]( + { + error, + idempotencyKey: "step:attempt-0", + scope: ATTEMPT_SCOPE, + type, + }, + providerContext(), + ); + } + + const spans = (await backgroundLogger.drain()) as Array< + Record + >; + const step = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(step?.metrics?.end).toEqual(expect.any(Number)); + if (error !== undefined) { + expect(step?.error).toContain("attempt exploded"); + } + }); +}); diff --git a/js/src/instrumentation/providers/eve-instrumentation.ts b/js/src/instrumentation/providers/eve-instrumentation.ts new file mode 100644 index 000000000..8729c6a48 --- /dev/null +++ b/js/src/instrumentation/providers/eve-instrumentation.ts @@ -0,0 +1,1201 @@ +import { debugLogger } from "../../debug-logger"; +import { + Attachment, + NOOP_SPAN, + _internalStartSpanWithInitialMerge, + currentLogger, + flush, + updateSpan, + withCurrent, +} from "../../logger"; +import type { Span } from "../../logger"; +import { LRUCache } from "../../lru-cache"; +import { + INSTRUMENTATION_NAMES, + withSpanInstrumentationName, +} from "../../span-origin"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { getCurrentUnixTimestamp } from "../../util"; +import { + convertDataToBlob, + getExtensionFromMediaType, + processInputAttachments, +} from "../../wrappers/attachment-utils"; +import type { + EveInstrumentationActionCompletedEvent, + EveInstrumentationActionFailedEvent, + EveInstrumentationActionStartedEvent, + EveInstrumentationContentPart, + EveInstrumentationDefinition, + EveInstrumentationHandlerContext, + EveInstrumentationModelCallCompletedEvent, + EveInstrumentationModelCallFailedEvent, + EveInstrumentationModelCallStartedEvent, + EveInstrumentationModelInput, + EveInstrumentationStepAttemptCompletedEvent, + EveInstrumentationStepAttemptFailedEvent, + EveInstrumentationTurnFailedEvent, + EveInstrumentationTurnSettledEvent, + EveInstrumentationTurnStartedEvent, + EveInstrumentationUsage, +} from "../../vendor-sdk-types/eve"; + +type EveSpan = Pick; + +type EveSpanReference = { + readonly exported: string; + readonly rootSpanId: string; + readonly spanId: string; + readonly turnMetricContributions?: Record>; +}; + +type TurnState = { + key: string; + metadata: Record; + rootSpanId: string; + rowId: string; + span: EveSpan; + spanId: string; +}; + +type EveInstrumentationEvent = + | EveInstrumentationActionCompletedEvent + | EveInstrumentationActionFailedEvent + | EveInstrumentationActionStartedEvent + | EveInstrumentationModelCallCompletedEvent + | EveInstrumentationModelCallFailedEvent + | EveInstrumentationModelCallStartedEvent + | EveInstrumentationStepAttemptCompletedEvent + | EveInstrumentationStepAttemptFailedEvent + | EveInstrumentationTurnFailedEvent + | EveInstrumentationTurnSettledEvent + | EveInstrumentationTurnStartedEvent; + +const MAX_EVE_CACHE_ENTRIES = 10_000; +const eveRootSpanIdsByTurnKey = new LRUCache({ + max: MAX_EVE_CACHE_ENTRIES, +}); + +/** + * Creates an Eve instrumentation provider. + * + * Eve 0.34.0 introduced the provider event API used here. Older Eve hook and + * single-file instrumentation layouts are intentionally unsupported. + */ +export function braintrustEveInstrumentation( + options: { + metadata?: Record; + setup?: EveInstrumentationDefinition["setup"]; + } = {}, +): EveInstrumentationDefinition { + const bridge = new EveBridge(options.metadata); + return { + capture: "content", + events: { + "action.completed": (event, context) => bridge.handle(event, context), + "action.failed": (event, context) => bridge.handle(event, context), + "action.started": (event, context) => bridge.handle(event, context), + "model.call.completed": (event, context) => bridge.handle(event, context), + "model.call.failed": (event, context) => bridge.handle(event, context), + "model.call.started": (event, context) => bridge.handle(event, context), + "step.attempt.completed": (event, context) => + bridge.handle(event, context), + "step.attempt.failed": (event, context) => bridge.handle(event, context), + "turn.cancelled": (event, context) => bridge.handle(event, context), + "turn.completed": (event, context) => bridge.handle(event, context), + "turn.failed": (event, context) => bridge.handle(event, context), + "turn.started": (event, context) => bridge.handle(event, context), + }, + flush: () => flush(), + setup: options.setup, + }; +} + +class ResumedEveSpan implements EveSpan { + private endTime: number | undefined; + + constructor(private readonly reference: EveSpanReference) {} + + get rootSpanId(): string { + return this.reference.rootSpanId; + } + + get spanId(): string { + return this.reference.spanId; + } + + log(event: Parameters[0]): void { + updateSpan({ + exported: this.reference.exported, + ...event, + }); + } + + end(args?: Parameters[0]): number { + if (this.endTime === undefined) { + this.endTime = args?.endTime ?? getCurrentUnixTimestamp(); + this.log({ metrics: { end: this.endTime } }); + } + return this.endTime; + } +} + +class EveBridge { + private turnsByKey = new LRUCache({ + max: MAX_EVE_CACHE_ENTRIES, + }); + private turnsStartingByKey = new Map>(); + private openModelsByAttempt = new Map>(); + private turnMetricContributionsByKey = new Map< + string, + Record> + >(); + + constructor(private readonly configuredMetadata?: Record) {} + + async handle( + event: EveInstrumentationEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + try { + switch (event.type) { + case "turn.started": + await this.getOrStartTurn(event, context); + return; + case "model.call.started": + await this.handleModelCallStarted(event, context); + return; + case "model.call.completed": + await this.handleModelCallCompleted(event, context); + return; + case "model.call.failed": + await this.handleModelCallFailed(event, context); + return; + case "step.attempt.completed": + case "step.attempt.failed": + this.handleStepAttemptTerminal(event); + return; + case "action.started": + await this.handleActionStarted(event, context); + return; + case "action.completed": + await this.handleActionCompleted(event, context); + return; + case "action.failed": + await this.handleActionFailed(event, context); + return; + case "turn.cancelled": + case "turn.completed": + await this.handleTurnSettled(event, context); + return; + case "turn.failed": + await this.handleTurnFailed(event, context); + return; + } + } catch (error) { + debugLogger.warn("Error in Eve instrumentation provider:", error); + } + } + + private async handleModelCallStarted( + event: EveInstrumentationModelCallStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const metadata = { + ...this.metadata(event.scope.sessionId), + ...modelMetadata(event.model.modelId, event.model.provider), + }; + const input = capturedModelInput(event.input); + const ids = await generateEveIds("step", event.idempotencyKey); + const span = await this.startEveSpan(context, { + event: { + id: ids.rowId, + ...(input === undefined ? {} : { input }), + metadata, + }, + name: "eve.step", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.LLM }, + spanId: ids.spanId, + }); + const reference = readSpanReference(context.state.get()); + if (reference !== undefined) { + context.state.set({ + ...reference, + turnMetricContributions: + this.turnMetricContributionsByKey.get(turn.key) ?? {}, + }); + } + const openModels = + this.openModelsByAttempt.get(event.scope.attemptId) ?? new Map(); + openModels.set(event.idempotencyKey, span); + this.openModelsByAttempt.set(event.scope.attemptId, openModels); + + if (input !== undefined) { + const initialInput = initialTurnInput(input); + if (initialInput !== undefined) { + this.updateTurn(turn, { input: initialInput }); + } + } + } + + private async handleModelCallCompleted( + event: EveInstrumentationModelCallCompletedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + this.restoreTurnMetricContributions(turn.key, context.state.get()); + const span = await this.resumeModelSpan(event, context, turn); + const metrics = usageMetrics(event.usage); + const output = modelOutput(event.finishReason, event.content); + span.log({ metrics, output }); + span.end(); + this.closeOpenModel(event.scope.attemptId, event.idempotencyKey); + this.updateTurnMetrics(turn, event.idempotencyKey, metrics); + + const finalOutput = finalText(event.content); + if ( + finalOutput !== undefined && + normalizedFinishReason(event.finishReason) !== "tool_calls" + ) { + this.updateTurn(turn, { output: finalOutput }); + } + } + + private async handleModelCallFailed( + event: EveInstrumentationModelCallFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const span = await this.resumeModelSpan(event, context, turn); + span.log({ error: toError(event.error, "Eve model call failed") }); + span.end(); + this.closeOpenModel(event.scope.attemptId, event.idempotencyKey); + } + + private handleStepAttemptTerminal( + event: + | EveInstrumentationStepAttemptCompletedEvent + | EveInstrumentationStepAttemptFailedEvent, + ): void { + const openModels = this.openModelsByAttempt.get(event.scope.attemptId); + if (openModels === undefined) { + return; + } + for (const span of openModels.values()) { + if (event.type === "step.attempt.failed") { + span.log({ + error: toError(event.error, "Eve step attempt failed"), + }); + } + span.end(); + } + this.openModelsByAttempt.delete(event.scope.attemptId); + } + + private async handleActionStarted( + event: EveInstrumentationActionStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const metadata = turn.metadata; + const input = processInputAttachments(event.input); + const ids = await generateEveIds( + "action", + event.scope.sessionId, + event.scope.turnId, + event.callId, + ); + await this.startEveSpan(context, { + event: { + id: ids.rowId, + ...(input === undefined ? {} : { input }), + metadata, + }, + name: event.name, + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + spanId: ids.spanId, + }); + } + + private async handleActionCompleted( + event: EveInstrumentationActionCompletedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const span = await this.resumeActionSpan(event, context); + if (event.output.type === "error") { + span.log({ + error: toError(event.output.error, "Eve action failed"), + }); + } else if (event.output.output !== undefined) { + span.log({ output: processInputAttachments(event.output.output) }); + } + span.end(actionEndArgs(event.acceptedAtMs)); + } + + private async handleActionFailed( + event: EveInstrumentationActionFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const span = await this.resumeActionSpan(event, context); + span.log({ + error: toError( + event.error, + event.errorCode === undefined + ? "Eve action " + event.outcome + : event.errorCode + ": Eve action " + event.outcome, + ), + }); + span.end(actionEndArgs(event.acceptedAtMs)); + } + + private async handleTurnSettled( + event: EveInstrumentationTurnSettledEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.resumeTurn(event, context); + turn.span.end(); + this.turnsByKey.delete(turn.key); + this.turnMetricContributionsByKey.delete(turn.key); + } + + private async handleTurnFailed( + event: EveInstrumentationTurnFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.resumeTurn(event, context); + turn.span.log({ + error: toError(event.error, "Eve turn failed"), + }); + turn.span.end(); + this.turnsByKey.delete(turn.key); + this.turnMetricContributionsByKey.delete(turn.key); + } + + private async startTurnSpan( + event: EveInstrumentationTurnStartedEvent, + context: EveInstrumentationHandlerContext, + metadata: Record, + ): Promise { + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + const parentSpanId = + event.parentLineage === undefined + ? undefined + : await deterministicEveId( + "eve:action", + event.parentLineage.sessionId, + event.parentLineage.turnId, + event.parentLineage.callId, + ); + const parentTurn = + event.parentLineage === undefined + ? undefined + : this.turnsByKey.get( + turnKey(event.parentLineage.sessionId, event.parentLineage.turnId), + ); + const parentRootSpanId = + event.parentLineage === undefined + ? undefined + : eveRootSpanIdsByTurnKey.get( + turnKey(event.parentLineage.sessionId, event.parentLineage.turnId), + ); + const rootSpanId = + event.parentLineage === undefined + ? await deterministicEveId("eve:root", event.sessionId, event.turnId) + : (parentTurn?.rootSpanId ?? + parentRootSpanId ?? + (typeof event.parentTraceContext?.traceId === "string" && + event.parentTraceContext.traceId.length > 0 + ? event.parentTraceContext.traceId + : await deterministicEveId( + "eve:root", + event.parentLineage.sessionId, + event.parentLineage.turnId, + ))); + return await this.startEveSpan(context, { + event: { id: ids.rowId, metadata }, + name: "eve.turn", + parentSpanIds: + parentSpanId === undefined + ? { parentSpanIds: [], rootSpanId } + : { rootSpanId, spanId: parentSpanId }, + spanAttributes: { type: SpanTypeAttribute.TASK }, + spanId: ids.spanId, + }); + } + + private async startEveSpan( + context: EveInstrumentationHandlerContext, + args: Parameters[0], + ): Promise { + const reference = readSpanReference(context.state.get()); + if (reference !== undefined) { + return new ResumedEveSpan(reference); + } + + const startTime = args?.startTime ?? getCurrentUnixTimestamp(); + const span = withCurrent(NOOP_SPAN, () => + _internalStartSpanWithInitialMerge( + withSpanInstrumentationName( + { ...args, startTime }, + INSTRUMENTATION_NAMES.EVE, + ), + ), + ); + + try { + const reference: EveSpanReference = { + exported: await span.export(), + rootSpanId: span.rootSpanId, + spanId: span.spanId, + }; + context.state.set(reference); + } catch (error) { + debugLogger.warn("Error exporting Eve span for resumption:", error); + } + return span; + } + + private async getOrStartTurn( + event: EveInstrumentationTurnStartedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(event.sessionId, event.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; + } + + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; + } + + const metadata = this.metadata(event.sessionId); + const start = (async () => { + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + const span = await this.startTurnSpan(event, context, metadata); + const turn = { + key, + metadata, + rootSpanId: span.rootSpanId, + rowId: ids.rowId, + span, + spanId: span.spanId, + }; + this.turnsByKey.set(key, turn); + eveRootSpanIdsByTurnKey.set(key, turn.rootSpanId); + return turn; + })(); + this.turnsStartingByKey.set(key, start); + try { + return await start; + } finally { + if (this.turnsStartingByKey.get(key) === start) { + this.turnsStartingByKey.delete(key); + } + } + } + + private async resumeTurn( + event: + | EveInstrumentationTurnSettledEvent + | EveInstrumentationTurnFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(event.sessionId, event.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; + } + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; + } + + const metadata = this.metadata(event.sessionId); + const span = await this.startTurnSpan( + { + idempotencyKey: event.idempotencyKey, + rootSessionId: event.sessionId, + sequence: 0, + sessionId: event.sessionId, + turnId: event.turnId, + type: "turn.started", + }, + context, + metadata, + ); + const ids = await generateEveIds("turn", event.sessionId, event.turnId); + return { + key, + metadata, + rootSpanId: span.rootSpanId, + rowId: ids.rowId, + span, + spanId: span.spanId, + }; + } + + private async turnForScope( + scope: EveInstrumentationModelCallStartedEvent["scope"], + context: EveInstrumentationHandlerContext, + ): Promise { + const key = turnKey(scope.sessionId, scope.turnId); + const existing = this.turnsByKey.get(key); + if (existing !== undefined) { + return existing; + } + const starting = this.turnsStartingByKey.get(key); + if (starting !== undefined) { + return await starting; + } + + const ids = await generateEveIds("turn", scope.sessionId, scope.turnId); + const persistedRootSpanId = readSpanReference( + context.state.get(), + )?.rootSpanId; + const rememberedRootSpanId = eveRootSpanIdsByTurnKey.get(key); + return { + key, + metadata: this.metadata(scope.sessionId), + rootSpanId: + persistedRootSpanId ?? + rememberedRootSpanId ?? + (await deterministicEveId( + "eve:root", + scope.rootSessionId ?? scope.sessionId, + scope.turnId, + )), + rowId: ids.rowId, + span: NOOP_SPAN, + spanId: ids.spanId, + }; + } + + private updateTurn( + turn: TurnState, + event: { input?: unknown; output?: unknown }, + ): void { + currentLogger()?.updateSpan({ + id: turn.rowId, + root_span_id: turn.rootSpanId, + span_id: turn.spanId, + ...event, + }); + } + + private closeOpenModel(attemptId: string, idempotencyKey: string): void { + const openModels = this.openModelsByAttempt.get(attemptId); + if (openModels === undefined) { + return; + } + openModels.delete(idempotencyKey); + if (openModels.size === 0) { + this.openModelsByAttempt.delete(attemptId); + } + } + + private restoreTurnMetricContributions( + turnKey: string, + state: unknown, + ): void { + const persisted = readSpanReference(state)?.turnMetricContributions; + if ( + persisted === undefined || + this.turnMetricContributionsByKey.has(turnKey) + ) { + return; + } + this.turnMetricContributionsByKey.set(turnKey, { ...persisted }); + } + + private updateTurnMetrics( + turn: TurnState, + idempotencyKey: string, + metrics: Record, + ): void { + const contributions = { + ...(this.turnMetricContributionsByKey.get(turn.key) ?? {}), + [idempotencyKey]: metrics, + }; + this.turnMetricContributionsByKey.set(turn.key, contributions); + const totals: Record = {}; + for (const contribution of Object.values(contributions)) { + for (const [name, value] of Object.entries(contribution)) { + totals[name] = (totals[name] ?? 0) + value; + } + } + if (Object.keys(totals).length === 0) { + return; + } + currentLogger()?.updateSpan({ + id: turn.rowId, + metrics: totals, + root_span_id: turn.rootSpanId, + span_id: turn.spanId, + }); + } + + private async resumeModelSpan( + event: + | EveInstrumentationModelCallCompletedEvent + | EveInstrumentationModelCallFailedEvent, + context: EveInstrumentationHandlerContext, + turn: TurnState, + ): Promise { + const ids = await generateEveIds("step", event.idempotencyKey); + return await this.startEveSpan(context, { + event: { + id: ids.rowId, + metadata: this.metadata(event.scope.sessionId), + }, + name: "eve.step", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.LLM }, + spanId: ids.spanId, + }); + } + + private async resumeActionSpan( + event: + | EveInstrumentationActionCompletedEvent + | EveInstrumentationActionFailedEvent, + context: EveInstrumentationHandlerContext, + ): Promise { + const turn = await this.turnForScope(event.scope, context); + const separator = event.idempotencyKey.lastIndexOf(":"); + const callId = + separator < 0 + ? event.idempotencyKey + : event.idempotencyKey.slice(separator + 1); + const ids = await generateEveIds( + "action", + event.scope.sessionId, + event.scope.turnId, + callId, + ); + return await this.startEveSpan(context, { + event: { id: ids.rowId, metadata: turn.metadata }, + name: "eve.action", + parentSpanIds: { + rootSpanId: turn.rootSpanId, + spanId: turn.spanId, + }, + spanAttributes: { type: SpanTypeAttribute.TOOL }, + spanId: ids.spanId, + }); + } + + private metadata(sessionId: string): Record { + return { + ...(this.configuredMetadata ?? {}), + "eve.session_id": sessionId, + }; + } +} + +function readSpanReference(value: unknown): EveSpanReference | undefined { + if (!isObject(value)) { + return undefined; + } + const exported = value["exported"]; + const rootSpanId = value["rootSpanId"]; + const spanId = value["spanId"]; + if ( + typeof exported !== "string" || + typeof rootSpanId !== "string" || + typeof spanId !== "string" + ) { + return undefined; + } + return { + exported, + rootSpanId, + spanId, + ...(isObject(value["turnMetricContributions"]) + ? { + turnMetricContributions: numericMetricContributions( + value["turnMetricContributions"], + ), + } + : {}), + }; +} + +function capturedModelInput( + input: EveInstrumentationModelInput | undefined, +): unknown[] | undefined { + if (input === undefined) { + return undefined; + } + const value: unknown[] = []; + if (typeof input.instructions === "string") { + value.push({ content: input.instructions, role: "system" }); + } else if (Array.isArray(input.instructions)) { + for (const instruction of input.instructions) { + const projected = projectedModelMessage(instruction); + if (projected?.role === "system") { + value.push(projected); + } + } + } else if (isObject(input.instructions)) { + const instruction = projectedModelMessage(input.instructions); + if (instruction?.role === "system") { + value.push(instruction); + } + } + for (const message of input.messages) { + const projected = projectedModelMessage(message); + if (projected !== undefined) { + value.push(projected); + } + } + return value; +} + +function projectedModelMessage( + value: unknown, +): { content: unknown; role: string } | undefined { + if (!isObject(value)) { + return undefined; + } + const role = value["role"]; + if ( + role !== "system" && + role !== "user" && + role !== "assistant" && + role !== "tool" + ) { + return undefined; + } + const content = value["content"]; + if (typeof content === "string") { + return { content, role }; + } + if (!Array.isArray(content)) { + return undefined; + } + const projected: unknown[] = []; + for (const part of content) { + if (!isObject(part) || typeof part["type"] !== "string") { + continue; + } + switch (part["type"]) { + case "text": + case "reasoning": + if (typeof part["text"] === "string") { + projected.push({ text: part["text"], type: part["type"] }); + } + break; + case "image": { + if (part["image"] === undefined) { + break; + } + const mediaType = + typeof part["mediaType"] === "string" + ? part["mediaType"] + : "image/png"; + projected.push({ + image: mediaAttachment(part["image"], mediaType, undefined), + ...(typeof part["mediaType"] === "string" ? { mediaType } : {}), + type: "image", + }); + break; + } + case "file": { + const mediaType = + typeof part["mediaType"] === "string" + ? part["mediaType"] + : "application/octet-stream"; + const data = + part["data"] === undefined + ? undefined + : mediaAttachment( + part["data"], + mediaType, + typeof part["filename"] === "string" + ? part["filename"] + : undefined, + ); + projected.push({ + ...(data === undefined ? {} : { data }), + ...(typeof part["filename"] === "string" + ? { filename: part["filename"] } + : {}), + mediaType, + type: "file", + }); + break; + } + case "tool-call": + if ( + typeof part["toolCallId"] === "string" && + typeof part["toolName"] === "string" + ) { + projected.push({ + input: part["input"], + toolCallId: part["toolCallId"], + toolName: part["toolName"], + type: "tool-call", + }); + } + break; + case "tool-result": + if ( + typeof part["toolCallId"] === "string" && + typeof part["toolName"] === "string" + ) { + projected.push({ + output: projectedToolOutput(part["output"]), + toolCallId: part["toolCallId"], + toolName: part["toolName"], + type: "tool-result", + }); + } + break; + case "tool-approval-response": + if ( + typeof part["approvalId"] === "string" && + typeof part["approved"] === "boolean" + ) { + projected.push({ + approvalId: part["approvalId"], + approved: part["approved"], + ...(typeof part["reason"] === "string" + ? { reason: part["reason"] } + : {}), + type: "tool-approval-response", + }); + } + break; + default: + break; + } + } + return { content: projected, role }; +} + +function projectedToolOutput(value: unknown): unknown { + if (!isObject(value)) { + return value; + } + if (typeof value["type"] !== "string") { + return undefined; + } + switch (value["type"]) { + case "text": + case "json": + case "error-text": + case "error-json": + return { type: value["type"], value: value["value"] }; + case "execution-denied": + return { + ...(typeof value["reason"] === "string" + ? { reason: value["reason"] } + : {}), + type: "execution-denied", + }; + case "content": { + if (!Array.isArray(value["value"])) { + return { type: "content", value: [] }; + } + const content: unknown[] = []; + for (const part of value["value"]) { + if (!isObject(part) || typeof part["type"] !== "string") { + continue; + } + if (part["type"] === "text" && typeof part["text"] === "string") { + content.push({ text: part["text"], type: "text" }); + } else if ( + (part["type"] === "file-data" || part["type"] === "image-data") && + part["data"] !== undefined && + typeof part["mediaType"] === "string" + ) { + const data = mediaAttachment( + part["data"], + part["mediaType"], + typeof part["filename"] === "string" ? part["filename"] : undefined, + ); + content.push({ + ...(data === undefined ? {} : { data }), + ...(typeof part["filename"] === "string" + ? { filename: part["filename"] } + : {}), + mediaType: part["mediaType"], + type: part["type"], + }); + } else if ( + (part["type"] === "file-url" || part["type"] === "image-url") && + typeof part["url"] === "string" + ) { + content.push({ type: part["type"], url: part["url"] }); + } + } + return { + type: "content", + value: content, + }; + } + default: + return undefined; + } +} + +function mediaAttachment( + data: unknown, + contentType: string, + filename: string | undefined, +): unknown { + const blob = convertDataToBlob(data, contentType); + if (blob === null) { + return data; + } + return new Attachment({ + contentType, + data: blob, + filename: + filename ?? `attachment.${getExtensionFromMediaType(contentType)}`, + }); +} + +function initialTurnInput(input: readonly unknown[]): unknown[] | undefined { + for (let index = input.length - 1; index >= 0; index--) { + const message = input[index]; + if (isObject(message) && message["role"] === "user") { + return [message]; + } + } + return undefined; +} + +function modelOutput( + finishReason: string, + content: readonly EveInstrumentationContentPart[] | undefined, +): unknown { + const parts = content ?? []; + const text = parts + .filter( + ( + part, + ): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + const reasoning = parts + .filter( + ( + part, + ): part is Extract< + EveInstrumentationContentPart, + { type: "reasoning" } + > => part.type === "reasoning", + ) + .map((part) => ({ content: part.text })); + const toolCalls = parts.flatMap((part) => { + if (part.type !== "tool-call") { + return []; + } + return [ + { + function: { + arguments: jsonString(part.input), + name: part.toolName, + }, + id: part.callId, + type: "function", + }, + ]; + }); + return [ + { + finish_reason: normalizedFinishReason(finishReason), + index: 0, + message: { + content: text.length === 0 ? null : text, + ...(reasoning.length === 0 ? {} : { reasoning }), + role: "assistant", + ...(toolCalls.length === 0 ? {} : { tool_calls: toolCalls }), + }, + }, + ]; +} + +function finalText( + content: readonly EveInstrumentationContentPart[] | undefined, +): string | undefined { + const text = (content ?? []) + .filter( + ( + part, + ): part is Extract => + part.type === "text", + ) + .map((part) => part.text) + .join(""); + return text.length === 0 ? undefined : text; +} + +function jsonString(value: unknown): string { + try { + return JSON.stringify(value) ?? "null"; + } catch { + return "null"; + } +} + +function normalizedFinishReason(finishReason: string): string { + if (finishReason === "content-filter") { + return "content_filter"; + } + if (finishReason === "tool-calls") { + return "tool_calls"; + } + return finishReason; +} + +function usageMetrics(usage: EveInstrumentationUsage): Record { + const inputTokens = validMetric(usage.inputTokens); + const outputTokens = validMetric(usage.outputTokens); + const cacheReadTokens = validMetric(usage.inputTokenDetails?.cacheReadTokens); + const cacheWriteTokens = validMetric( + usage.inputTokenDetails?.cacheWriteTokens, + ); + return { + ...(inputTokens === undefined ? {} : { prompt_tokens: inputTokens }), + ...(outputTokens === undefined ? {} : { completion_tokens: outputTokens }), + ...(inputTokens === undefined || outputTokens === undefined + ? {} + : { tokens: inputTokens + outputTokens }), + ...(cacheReadTokens === undefined + ? {} + : { prompt_cached_tokens: cacheReadTokens }), + ...(cacheWriteTokens === undefined + ? {} + : { prompt_cache_creation_tokens: cacheWriteTokens }), + }; +} + +function validMetric(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : undefined; +} + +function numericMetrics( + value: Record, +): Record { + const metrics: Record = {}; + for (const name of [ + "completion_tokens", + "prompt_cache_creation_tokens", + "prompt_cached_tokens", + "prompt_tokens", + "tokens", + ]) { + const metric = validMetric(value[name]); + if (metric !== undefined) { + metrics[name] = metric; + } + } + return metrics; +} + +function numericMetricContributions( + value: Record, +): Record> { + const contributions: Record> = {}; + for (const [idempotencyKey, contribution] of Object.entries(value)) { + if (isObject(contribution)) { + contributions[idempotencyKey] = numericMetrics(contribution); + } + } + return contributions; +} + +function modelMetadata( + modelId: string, + providerId: string, +): Record { + const provider = providerId.split(".")[0]?.trim(); + const model = modelId.trim(); + return { + ...(model.length === 0 ? {} : { model }), + ...(provider === undefined || provider.length === 0 ? {} : { provider }), + }; +} + +function toError(error: unknown, fallback: string): Error { + if (error instanceof Error) { + return error; + } + if (typeof error === "string" && error.length > 0) { + return new Error(error); + } + if (isObject(error) && typeof error["message"] === "string") { + const code = typeof error["code"] === "string" ? error["code"] + ": " : ""; + const result = new Error(code + error["message"]); + result.cause = error; + return result; + } + const result = new Error(fallback); + if (error !== undefined) { + result.cause = error; + } + return result; +} + +function actionEndArgs( + acceptedAtMs: number | undefined, +): { endTime: number } | undefined { + return typeof acceptedAtMs === "number" && Number.isFinite(acceptedAtMs) + ? { endTime: acceptedAtMs / 1000 } + : undefined; +} + +function turnKey(sessionId: string, turnId: string): string { + return sessionId + ":" + turnId; +} + +async function generateEveIds( + kind: "action" | "step" | "turn", + ...parts: string[] +): Promise<{ rowId: string; spanId: string }> { + const [rowId, spanId] = await Promise.all([ + deterministicEveId("eve:row:" + kind, ...parts), + deterministicEveId("eve:" + kind, ...parts), + ]); + return { rowId, spanId }; +} + +async function deterministicEveId(...parts: string[]): Promise { + const data = new TextEncoder().encode( + parts.map((part) => part.length + ":" + part).join("\0"), + ); + const digest = await globalThis.crypto.subtle.digest("SHA-256", data); + const bytes = Array.from(new Uint8Array(digest, 0, 16)); + const hex = bytes.map((byte) => byte.toString(16).padStart(2, "0")).join(""); + return ( + hex.slice(0, 8) + + "-" + + hex.slice(8, 12) + + "-" + + hex.slice(12, 16) + + "-" + + hex.slice(16, 20) + + "-" + + hex.slice(20) + ); +} diff --git a/js/src/instrumentation/plugins/flue-plugin.test.ts b/js/src/instrumentation/providers/flue-instrumentation.test.ts similarity index 68% rename from js/src/instrumentation/plugins/flue-plugin.test.ts rename to js/src/instrumentation/providers/flue-instrumentation.test.ts index 071b2a1b4..ae6fbec8b 100644 --- a/js/src/instrumentation/plugins/flue-plugin.test.ts +++ b/js/src/instrumentation/providers/flue-instrumentation.test.ts @@ -37,43 +37,6 @@ vi.mock("../../debug-logger", () => ({ }, })); -const { mockNewTracingChannel, mockTracingChannels } = vi.hoisted(() => { - const tracingChannels = new Map(); - - function tracingChannel(name: string) { - const existing = tracingChannels.get(name); - if (existing) { - return existing; - } - - const handlers = new Set(); - const stores = new Map unknown>(); - const channel = { - __handlers: handlers, - __stores: stores, - start: { - bindStore: vi.fn( - (store: unknown, transform: (message: any) => unknown) => { - stores.set(store, transform); - }, - ), - unbindStore: vi.fn((store: unknown) => stores.delete(store)), - }, - subscribe: vi.fn((handler: any) => { - handlers.add(handler); - }), - unsubscribe: vi.fn((handler: any) => handlers.delete(handler)), - }; - tracingChannels.set(name, channel); - return channel; - } - - return { - mockNewTracingChannel: vi.fn((name: string) => tracingChannel(name)), - mockTracingChannels: tracingChannels, - }; -}); - vi.mock("../../logger", () => ({ BRAINTRUST_CURRENT_SPAN_STORE: mockCurrentSpanStoreSymbol, NOOP_SPAN: {}, @@ -101,24 +64,9 @@ vi.mock("../../logger", () => ({ }, })); -vi.mock("../../isomorph", () => ({ - default: { - newTracingChannel: mockNewTracingChannel, - }, -})); - -import { - FluePlugin, - braintrustFlueInstrumentation, - braintrustFlueObserver, -} from "./flue-plugin"; - -type Subscriber = typeof braintrustFlueObserver; - -const CREATE_CONTEXT_CHANNEL_NAME = - "orchestrion:@flue/runtime:createFlueContext"; +import { braintrustFlueInstrumentation } from "./flue-instrumentation"; -describe("Flue observe instrumentation", () => { +describe("Flue instrumentation", () => { let spans: Array<{ args: any; end: ReturnType; @@ -160,52 +108,20 @@ describe("Flue observe instrumentation", () => { }); afterEach(() => { - delete (globalThis as Record)[ - Symbol.for("braintrust.flue.auto-state") - ]; delete (globalThis as Record)[ Symbol.for("braintrust.flue.observe-bridge") ]; - for (const channel of mockTracingChannels.values()) { - channel.__handlers.clear(); - channel.__stores.clear(); - } vi.clearAllMocks(); }); - it("exports a subscriber that can be passed directly to Flue observe", () => { - const subscribers: Subscriber[] = []; - const unsubscribe = vi.fn(); - const observe = vi.fn((subscriber: Subscriber) => { - subscribers.push(subscriber); - return unsubscribe; - }); - - const unregister = observe(braintrustFlueObserver); - - expect(observe).toHaveBeenCalledTimes(1); - expect(observe).toHaveBeenCalledWith(braintrustFlueObserver); - expect(subscribers).toHaveLength(1); - - subscribers[0]?.({ - runId: "run-1", - type: "run_start", - workflowName: "research", - }); - expect(findSpan("workflow:research")).toBeDefined(); - - unregister(); - expect(unsubscribe).toHaveBeenCalledTimes(1); - }); - - it("exports a Flue 1.0 instrumentation factory", async () => { + it("exports a Flue instrumentation factory", async () => { const instrumentation = braintrustFlueInstrumentation(); const instrument = vi.fn((value: typeof instrumentation) => value); const registered = instrument(instrumentation); expect(instrument).toHaveBeenCalledWith(instrumentation); - expect(registered.observe).toBe(braintrustFlueObserver); + expect(typeof registered.observe).toBe("function"); expect(registered.key).toBe(Symbol.for("braintrust.flue.instrumentation")); expect(typeof registered.interceptor).toBe("function"); expect(() => registered.dispose()).not.toThrow(); @@ -218,7 +134,7 @@ describe("Flue observe instrumentation", () => { type: "workflow", workflowName: "research", }, - { eventContext: { id: "ctx-1", runId: "run-1" } }, + { eventContext: { id: "ctx-1" } }, async () => mockStartSpan({ name: "app.phase" }), ); const workflowSpan = findSpan("workflow:research"); @@ -232,37 +148,7 @@ describe("Flue observe instrumentation", () => { expect("enterWith" in mockCurrentSpanStore).toBe(false); }); - it("keeps the legacy observer compatible with Flue 1.0 instrumentation", async () => { - expect(braintrustFlueObserver.observe).toBe(braintrustFlueObserver); - expect(braintrustFlueObserver.key).toBe( - Symbol.for("braintrust.flue.instrumentation"), - ); - expect(typeof braintrustFlueObserver.interceptor).toBe("function"); - expect(() => braintrustFlueObserver.dispose()).not.toThrow(); - - const appSpan = await braintrustFlueObserver.interceptor( - { - phase: "start", - runId: "run-1", - startedAt: "2026-05-27T05:12:31.000Z", - type: "workflow", - workflowName: "research", - }, - { eventContext: { id: "ctx-1", runId: "run-1" } }, - async () => mockStartSpan({ name: "app.phase" }), - ); - const workflowSpan = findSpan("workflow:research"); - - expect(workflowSpan).toBeDefined(); - expect(appSpan.spanParents).toEqual([workflowSpan?.spanId]); - expect(mockCurrentSpanStore.run).toHaveBeenCalledWith( - workflowSpan, - expect.any(Function), - ); - expect("enterWith" in mockCurrentSpanStore).toBe(false); - }); - - it("maps Flue 0.8 observe events into semantic Braintrust spans", () => { + it("maps Flue 1 workflow observations into semantic Braintrust spans", () => { const emit = observeEvents(); const usage = flueUsage(); const startedAt = "2026-05-27T05:12:31.000Z"; @@ -270,8 +156,7 @@ describe("Flue observe instrumentation", () => { emit( { instanceId: "instance-1", - owner: { kind: "workflow", workflowName: "research" }, - payload: { + input: { metadata: { scenario: "flue-instrumentation", testRunId: "e2e-run-1", @@ -284,7 +169,7 @@ describe("Flue observe instrumentation", () => { type: "run_start", workflowName: "research", }, - { id: "ctx-1", runId: "run-1" }, + { id: "ctx-1" }, ); emit({ operationId: "op-1", @@ -295,17 +180,19 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - api: "responses", - input: { - messages: [{ content: "Find Flue changes", role: "user" }], - systemPrompt: "Be precise", - tools: [{ name: "lookup", parameters: {} }], - }, - model: "claude-test", operationId: "op-1", - provider: "anthropic", purpose: "agent", - reasoning: "medium", + request: { + api: "responses", + input: { + messages: [{ content: "Find Flue changes", role: "user" }], + systemPrompt: "Be precise", + tools: [{ name: "lookup", parameters: {} }], + }, + providerName: "anthropic", + reasoningLevel: "medium", + requestedModel: "claude-test", + }, runId: "run-1", timestamp: "2026-05-27T05:12:33.000Z", turnId: "turn-1", @@ -331,23 +218,30 @@ describe("Flue observe instrumentation", () => { toolCallId: "tool-1", toolName: "lookup", turnId: "turn-1", - type: "tool_call", + type: "tool", }); emit({ - api: "responses", durationMs: 12, isError: false, - model: "claude-test", operationId: "op-1", - output: { content: [{ text: "done", type: "text" }], role: "assistant" }, - provider: "anthropic", purpose: "agent", + request: { + api: "responses", + providerName: "anthropic", + requestedModel: "claude-test", + }, + response: { + output: { + content: [{ text: "done", type: "text" }], + role: "assistant", + }, + finishReason: "stop", + usage, + }, runId: "run-1", - stopReason: "stop", timestamp: "2026-05-27T05:12:36.000Z", turnId: "turn-1", type: "turn", - usage, }); emit({ agent: "worker", @@ -493,150 +387,6 @@ describe("Flue observe instrumentation", () => { expect(mockFlush).toHaveBeenCalledTimes(1); }); - it("maps Flue 1.0 observations into semantic Braintrust spans", () => { - const emit = observeEvents(); - const usage = flueUsage(); - - emit( - { - eventIndex: 0, - input: { - metadata: { scenario: "flue-v1" }, - topic: "native instrumentation", - }, - runId: "run-1", - timestamp: "2026-05-27T05:12:31.000Z", - type: "run_start", - v: 3, - workflowName: "research", - }, - { id: "ctx-1", runId: "run-1" }, - ); - emit({ - eventIndex: 1, - operationId: "op-1", - operationKind: "prompt", - runId: "run-1", - type: "operation_start", - v: 3, - }); - emit({ - eventIndex: 2, - operationId: "op-1", - purpose: "agent", - request: { - api: "responses", - input: { - messages: [{ content: "Find native hooks", role: "user" }], - systemPrompt: "Be exact", - tools: [{ name: "lookup" }], - }, - model: "claude-test", - providerId: "anthropic", - providerName: "anthropic", - reasoning: "medium", - }, - runId: "run-1", - turnId: "turn-1", - type: "turn_request", - v: 3, - }); - emit({ - args: { query: "native flue instrumentation" }, - eventIndex: 3, - operationId: "op-1", - runId: "run-1", - toolCallId: "tool-1", - toolName: "lookup", - turnId: "turn-1", - type: "tool_start", - v: 3, - }); - emit({ - durationMs: 4, - eventIndex: 4, - isError: false, - operationId: "op-1", - output: { ok: true }, - runId: "run-1", - toolCallId: "tool-1", - toolName: "lookup", - turnId: "turn-1", - type: "tool", - v: 3, - }); - emit({ - durationMs: 12, - eventIndex: 5, - isError: false, - operationId: "op-1", - purpose: "agent", - request: { - api: "responses", - model: "claude-test", - providerId: "anthropic", - providerName: "anthropic", - }, - response: { - output: { content: [{ text: "done", type: "text" }] }, - stopReason: "stop", - usage, - }, - runId: "run-1", - turnId: "turn-1", - type: "turn", - v: 3, - }); - emit({ - durationMs: 50, - eventIndex: 6, - isError: false, - operationId: "op-1", - operationKind: "prompt", - result: { text: "PROMPT_DONE", usage }, - runId: "run-1", - type: "operation", - usage, - v: 3, - }); - - const workflowSpan = findSpan("workflow:research"); - const turnSpan = findSpan("flue.turn"); - const toolSpan = findSpan("tool:lookup"); - const operationSpan = findSpan("flue.prompt"); - - expect(workflowSpan?.args.event.input).toMatchObject({ - metadata: { scenario: "flue-v1" }, - topic: "native instrumentation", - }); - expect(turnSpan?.args.event).toMatchObject({ - input: [{ content: "Find native hooks", role: "user" }], - metadata: { - "flue.api": "responses", - "flue.model": "claude-test", - "flue.provider": "anthropic", - "flue.system_prompt": "Be exact", - provider: "anthropic", - reasoning: "medium", - tools: [{ name: "lookup" }], - }, - }); - expect(toolSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: { ok: true } }), - ); - expect(turnSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ - metadata: expect.objectContaining({ - "flue.stop_reason": "stop", - }), - output: { content: [{ text: "done", type: "text" }] }, - }), - ); - expect(operationSpan?.log).toHaveBeenCalledWith( - expect.objectContaining({ output: "PROMPT_DONE" }), - ); - }); - it("maps released Flue 2.0 observations and retains the final agent output", () => { const emit = observeEvents(); const usage = flueUsage(); @@ -790,59 +540,65 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - input: { - messages: [previousUser, previousAssistant, currentUser], - systemPrompt: "Be precise", - tools: [{ name: "lookup" }], - }, operationId: "op-1", purpose: "agent", + request: { + input: { + messages: [previousUser, previousAssistant, currentUser], + systemPrompt: "Be precise", + tools: [{ name: "lookup" }], + }, + }, runId: "run-1", turnId: "turn-1", type: "turn_request", }); emit({ operationId: "op-1", - output: toolCall, purpose: "agent", + response: { output: toolCall }, runId: "run-1", turnId: "turn-1", type: "turn", }); emit({ - input: { - messages: [ - previousUser, - previousAssistant, - currentUser, - toolCall, - toolResult, - ], - systemPrompt: "Be precise", - tools: [{ name: "lookup" }], - }, operationId: "op-1", purpose: "agent", + request: { + input: { + messages: [ + previousUser, + previousAssistant, + currentUser, + toolCall, + toolResult, + ], + systemPrompt: "Be precise", + tools: [{ name: "lookup" }], + }, + }, runId: "run-1", turnId: "turn-1", type: "turn_request", }); emit({ operationId: "op-1", - output: { content: "done", role: "assistant" }, purpose: "agent", + response: { output: { content: "done", role: "assistant" } }, runId: "run-1", turnId: "turn-1", type: "turn", }); emit({ - input: { - messages: [currentUser], - systemPrompt: "Use compacted context", - tools: [{ name: "search" }], - }, operationId: "op-1", purpose: "agent", + request: { + input: { + messages: [currentUser], + systemPrompt: "Use compacted context", + tools: [{ name: "search" }], + }, + }, runId: "run-1", turnId: "turn-1", type: "turn_request", @@ -927,13 +683,15 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - input: { - messages: [{ content: "finish", role: "user" }], - }, - model: "claude-test", operationId: "op-1", - provider: "anthropic", purpose: "agent", + request: { + input: { + messages: [{ content: "finish", role: "user" }], + }, + providerName: "anthropic", + requestedModel: "claude-test", + }, runId: "run-1", turnId: "turn-1", type: "turn_request", @@ -1008,13 +766,15 @@ describe("Flue observe instrumentation", () => { type: "compaction_start", }); emit({ - input: { - messages: [{ content: "summarize", role: "user" }], - }, - model: "gpt-test", operationId: "op-compact", - provider: "openai", purpose: "compaction_prefix", + request: { + input: { + messages: [{ content: "summarize", role: "user" }], + }, + providerName: "openai", + requestedModel: "gpt-test", + }, runId: "run-1", session: "main", turnId: "turn-compact", @@ -1220,7 +980,7 @@ describe("Flue observe instrumentation", () => { runId: "run-1", toolCallId: "tool-1", toolName: "lookup", - type: "tool_call", + type: "tool", }); expect(mockCurrentParentSpan.current).toBeUndefined(); @@ -1236,10 +996,10 @@ describe("Flue observe instrumentation", () => { type: "run_start", workflowName: "research", }, - { id: "ctx-1", runId: "run-1" }, + { id: "ctx-1" }, ); const workflowSpan = findSpan("workflow:research"); - const appSpan = await braintrustFlueObserver.interceptor( + const appSpan = await braintrustFlueInstrumentation().interceptor( { phase: "start", runId: "run-1", @@ -1247,7 +1007,7 @@ describe("Flue observe instrumentation", () => { type: "workflow", workflowName: "research", }, - { eventContext: { id: "ctx-1", runId: "run-1" } }, + { eventContext: { id: "ctx-1" } }, async () => mockStartSpan({ name: "app.phase" }), ); @@ -1283,7 +1043,7 @@ describe("Flue observe instrumentation", () => { }); const toolSpan = findSpan("tool:lookup"); - const appSpan = await braintrustFlueObserver.interceptor( + const appSpan = await braintrustFlueInstrumentation().interceptor( { toolCallId: "tool-1", toolName: "lookup", type: "tool" }, { operationId: "op-prompt", runId: "run-1" }, async () => mockStartSpan({ name: "app.tool-phase" }), @@ -1312,11 +1072,13 @@ describe("Flue observe instrumentation", () => { type: "operation_start", }); emit({ - input: { messages: [{ content: "hello", role: "user" }] }, - model: "claude-test", operationId: "op-prompt", - provider: "anthropic", purpose: "agent", + request: { + input: { messages: [{ content: "hello", role: "user" }] }, + providerName: "anthropic", + requestedModel: "claude-test", + }, runId: "run-1", turnId: "turn-1", type: "turn_request", @@ -1332,17 +1094,17 @@ describe("Flue observe instrumentation", () => { const operationSpan = findSpan("flue.prompt"); const turnSpan = findSpan("flue.turn"); const taskSpan = findSpan("flue.task"); - const agentAppSpan = await braintrustFlueObserver.interceptor( + const agentAppSpan = await braintrustFlueInstrumentation().interceptor( { operationId: "op-prompt", operationKind: "prompt", type: "agent" }, { operationId: "op-prompt", runId: "run-1" }, async () => mockStartSpan({ name: "app.agent-phase" }), ); - const modelAppSpan = await braintrustFlueObserver.interceptor( + const modelAppSpan = await braintrustFlueInstrumentation().interceptor( { turnId: "turn-1", type: "model" }, { operationId: "op-prompt", runId: "run-1", turnId: "turn-1" }, async () => mockStartSpan({ name: "app.model-phase" }), ); - const taskAppSpan = await braintrustFlueObserver.interceptor( + const taskAppSpan = await braintrustFlueInstrumentation().interceptor( { taskId: "task-1", type: "task" }, { operationId: "op-prompt", runId: "run-1", taskId: "task-1" }, async () => mockStartSpan({ name: "app.task-phase" }), @@ -1354,149 +1116,9 @@ describe("Flue observe instrumentation", () => { expect(mockCurrentParentSpan.current).toBeUndefined(); }); - it("subscribes transformed Flue contexts for auto instrumentation", () => { - const plugin = new FluePlugin(); - const contextSubscribers: Array<(event: unknown) => unknown> = []; - const unsubscribeContext = vi.fn(); - const context = { - id: "ctx-1", - runId: "run-1", - subscribeEvent: vi.fn((subscriber: (event: unknown) => unknown) => { - contextSubscribers.push(subscriber); - return unsubscribeContext; - }), - }; - - plugin.enable(); - expect(mockNewTracingChannel).toHaveBeenCalledWith( - CREATE_CONTEXT_CHANNEL_NAME, - ); - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).subscribe, - ).toHaveBeenCalledTimes(1); - - emitCreateContextEnd(context); - - expect(context.subscribeEvent).toHaveBeenCalledTimes(1); - contextSubscribers[0]?.({ - runId: "run-1", - type: "run_start", - workflowName: "auto-research", - }); - - expect( - findSpan("workflow:auto-research")?.args.event.metadata, - ).toMatchObject({ - "flue.context_id": "ctx-1", - "flue.context_run_id": "run-1", - }); - contextSubscribers[0]?.({ - durationMs: 1, - isError: false, - result: "done", - runId: "run-1", - type: "run_end", - }); - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - - plugin.disable(); - - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).unsubscribe, - ).toHaveBeenCalledTimes(1); - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - }); - - it("keeps auto instrumentation idempotent across plugin instances", () => { - const first = new FluePlugin(); - const second = new FluePlugin(); - const contextSubscribers: Array<(event: unknown) => unknown> = []; - const unsubscribeContext = vi.fn(); - const context = { - subscribeEvent: vi.fn((subscriber: (event: unknown) => unknown) => { - contextSubscribers.push(subscriber); - return unsubscribeContext; - }), - }; - - first.enable(); - second.enable(); - emitCreateContextEnd(context); - emitCreateContextEnd(context); - - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).subscribe, - ).toHaveBeenCalledTimes(1); - expect(context.subscribeEvent).toHaveBeenCalledTimes(1); - - first.disable(); - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).unsubscribe, - ).not.toHaveBeenCalled(); - expect(unsubscribeContext).not.toHaveBeenCalled(); - - second.disable(); - expect( - tracingChannel(CREATE_CONTEXT_CHANNEL_NAME).unsubscribe, - ).toHaveBeenCalledTimes(1); - - contextSubscribers[0]?.({ - runId: "run-after-disable", - type: "run_start", - workflowName: "after-disable", - }); - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - expect(findSpan("workflow:after-disable")).toBeUndefined(); - }); - - it("unsubscribes direct Flue contexts on terminal operation events", () => { - const plugin = new FluePlugin(); - const contextSubscribers: Array<(event: unknown) => unknown> = []; - const unsubscribeContext = vi.fn(); - const context = { - id: "direct-agent-1", - subscribeEvent: vi.fn((subscriber: (event: unknown) => unknown) => { - contextSubscribers.push(subscriber); - return unsubscribeContext; - }), - }; - - plugin.enable(); - emitCreateContextEnd(context); - contextSubscribers[0]?.({ - durationMs: 2, - instanceId: "direct-agent-1", - isError: false, - operationId: "op-1", - operationKind: "prompt", - result: "done", - type: "operation", - }); - - expect(unsubscribeContext).toHaveBeenCalledTimes(1); - expect(findSpan("flue.prompt")).toBeDefined(); - - plugin.disable(); - }); - function observeEvents() { return (event: unknown, ctx?: unknown) => - braintrustFlueObserver(event, ctx); - } - - function emitCreateContextEnd(result: unknown) { - for (const handlers of tracingChannel(CREATE_CONTEXT_CHANNEL_NAME) - .__handlers) { - handlers.end?.({ result }); - } - } - - function tracingChannel(channelName: string) { - const channel = mockTracingChannels.get(channelName); - if (!channel) { - throw new Error(`Missing mocked tracing channel: ${channelName}`); - } - return channel; + braintrustFlueInstrumentation().observe(event, ctx); } function findSpan(name: string) { diff --git a/js/src/instrumentation/plugins/flue-plugin.ts b/js/src/instrumentation/providers/flue-instrumentation.ts similarity index 80% rename from js/src/instrumentation/plugins/flue-plugin.ts rename to js/src/instrumentation/providers/flue-instrumentation.ts index 81de77992..1426afde6 100644 --- a/js/src/instrumentation/plugins/flue-plugin.ts +++ b/js/src/instrumentation/providers/flue-instrumentation.ts @@ -1,7 +1,5 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { debugLogger } from "../../debug-logger"; -import type { ChannelMessage } from "../core/channel-definitions"; -import type { IsoChannelHandlers } from "../../isomorph"; import { BRAINTRUST_CURRENT_SPAN_STORE, NOOP_SPAN, @@ -11,13 +9,11 @@ import { withCurrent, } from "../../logger"; import type { Span, StartSpanArgs } from "../../logger"; -import type { CurrentSpanStore } from "../../logger"; import { INSTRUMENTATION_NAMES, withSpanInstrumentationName, } from "../../span-origin"; import { SpanTypeAttribute } from "../../../util/index"; -import { flueChannels } from "./flue-channels"; import type { FlueBaseEvent, FlueCompactionEvent, @@ -28,7 +24,6 @@ import type { FlueExecutionInterceptor, FlueExecutionOperation, FlueInstrumentation, - FlueObservableContext, FlueOperationEvent, FlueOperationKind, FlueOperationStartEvent, @@ -44,20 +39,6 @@ import type { FlueTurnRequestEvent, } from "../../vendor-sdk-types/flue"; -type FlueObserver = (event: unknown, ctx?: unknown) => void; -type BraintrustFlueObserver = FlueObserver & FlueInstrumentation; - -type FlueAutoState = { - createContextChannel?: ReturnType< - typeof flueChannels.createContext.tracingChannel - >; - createContextHandlers?: IsoChannelHandlers< - ChannelMessage - >; - contexts: WeakSet; - refCount: number; -}; - type SpanState = { latestAgentOutput?: unknown; loggedInput?: boolean; @@ -73,7 +54,6 @@ type FlueTurnInputState = { toolsFingerprint?: string; }; -const FLUE_AUTO_STATE = Symbol.for("braintrust.flue.auto-state"); const FLUE_INSTRUMENTATION_KEY = Symbol.for("braintrust.flue.instrumentation"); const FLUE_OBSERVE_BRIDGE = Symbol.for("braintrust.flue.observe-bridge"); @@ -88,9 +68,9 @@ const interceptFlueExecution: FlueExecutionInterceptor = ( ) => getObserveBridge().intercept(operation, ctx, next); /** - * Manual instrumentation for flue. + * Instrumentation for flue. * - * This should be passed to flue's `instrument()` API if not using auto-instrumentation: `instrument(braintrustFlueInstrumentation())` + * Pass this to flue's `instrument()` API: `instrument(braintrustFlueInstrumentation())` */ export function braintrustFlueInstrumentation(): FlueInstrumentation { return { @@ -101,71 +81,6 @@ export function braintrustFlueInstrumentation(): FlueInstrumentation { }; } -/** - * Observer for flue pre version 1.0.0. - * - * This observer should be passed to flue's `observe()` API if not using auto-instrumentation. - */ -export const braintrustFlueObserver: BraintrustFlueObserver = Object.assign( - observeFlue, - braintrustFlueInstrumentation(), -); - -export class FluePlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push(enableFlueAutoInstrumentation()); - } - - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } -} - -function enableFlueAutoInstrumentation(): () => void { - const state = getAutoState(); - state.refCount += 1; - - if (!state.createContextHandlers) { - const createContextChannel = flueChannels.createContext.tracingChannel(); - const handlers: IsoChannelHandlers< - ChannelMessage - > = { - end: (event) => { - subscribeToFlueContext(event.result, state); - }, - }; - - createContextChannel.subscribe(handlers); - state.createContextChannel = createContextChannel; - state.createContextHandlers = handlers; - } - - let released = false; - return () => { - if (released) { - return; - } - released = true; - releaseAutoState(state); - }; -} - -function getAutoState(): FlueAutoState { - const existing = Reflect.get(globalThis, FLUE_AUTO_STATE); - if (isAutoState(existing)) { - return existing; - } - const state: FlueAutoState = { - contexts: new WeakSet(), - refCount: 0, - }; - Reflect.set(globalThis, FLUE_AUTO_STATE, state); - return state; -} - function getObserveBridge(): FlueObserveBridge { const existing = Reflect.get(globalThis, FLUE_OBSERVE_BRIDGE); if (isFlueObserveBridge(existing)) { @@ -180,94 +95,7 @@ function isFlueObserveBridge(value: unknown): value is FlueObserveBridge { return ( isObjectLike(value) && typeof Reflect.get(value, "handle") === "function" && - typeof Reflect.get(value, "reset") === "function" - ); -} - -function isAutoState(value: unknown): value is FlueAutoState { - return ( - isObjectLike(value) && - Reflect.get(value, "contexts") instanceof WeakSet && - typeof Reflect.get(value, "refCount") === "number" - ); -} - -function releaseAutoState(state: FlueAutoState): void { - state.refCount -= 1; - if (state.refCount > 0) { - return; - } - - try { - if (state.createContextChannel && state.createContextHandlers) { - state.createContextChannel.unsubscribe(state.createContextHandlers); - } - } finally { - Reflect.deleteProperty(globalThis, FLUE_AUTO_STATE); - } -} - -function subscribeToFlueContext(value: unknown, state: FlueAutoState): void { - if (!isObservableFlueContext(value) || state.contexts.has(value)) { - return; - } - - const ctx = flueContextFromUnknown(value); - let released = false; - let unsubscribe: (() => void) | undefined; - const release = () => { - if (released) { - return; - } - released = true; - try { - unsubscribe?.(); - } catch (error) { - logInstrumentationError("Flue context unsubscribe", error); - } - }; - - try { - unsubscribe = value.subscribeEvent((event) => { - if (state.refCount <= 0) { - release(); - return; - } - - braintrustFlueObserver(event, ctx); - if (isAutoContextTerminalEvent(event, ctx)) { - release(); - } - }); - state.contexts.add(value); - } catch (error) { - logInstrumentationError("Flue context subscription", error); - } -} - -function isAutoContextTerminalEvent( - event: unknown, - ctx: FlueContext | undefined, -): boolean { - if (!isObjectLike(event)) { - return false; - } - const type = Reflect.get(event, "type"); - if (type === "run_end") { - return true; - } - if (type !== "operation") { - return false; - } - return !ctx?.runId && typeof Reflect.get(event, "runId") !== "string"; -} - -function isObservableFlueContext( - value: unknown, -): value is FlueObservableContext { - return ( - isObjectLike(value) && - typeof Reflect.get(value, "subscribeEvent") === "function" + typeof Reflect.get(value, "intercept") === "function" ); } @@ -282,7 +110,6 @@ function isFlueEvent(event: object): event is FlueEvent { type === "turn_request" || type === "turn" || type === "tool_start" || - type === "tool_call" || type === "tool" || type === "task_start" || type === "task" || @@ -296,11 +123,7 @@ function flueContextFromUnknown(ctx: unknown): FlueContext | undefined { return undefined; } const id = Reflect.get(ctx, "id"); - const runId = Reflect.get(ctx, "runId"); - return { - ...(typeof id === "string" ? { id } : {}), - ...(typeof runId === "string" ? { runId } : {}), - }; + return typeof id === "string" ? { id } : undefined; } function isObjectLike(value: unknown): value is object { @@ -332,16 +155,6 @@ class FlueObserveBridge { } } - reset(): void { - this.compactionsByKey.clear(); - this.operationsById.clear(); - this.runsById.clear(); - this.seenEvents = new WeakSet(); - this.tasksById.clear(); - this.toolsByKey.clear(); - this.turnsByKey.clear(); - } - intercept( operation: FlueExecutionOperation, executionContext: FlueExecutionContext, @@ -389,7 +202,6 @@ class FlueObserveBridge { flueContextFromUnknown(executionContext.eventContext) ?? flueContextFromUnknown({ id: operation.workflowName, - runId: operation.runId, }); if (operation.phase === "resume") { this.handleRunResume( @@ -399,7 +211,6 @@ class FlueObserveBridge { startedAt: operation.startedAt, timestamp: new Date().toISOString(), type: "run_resume", - v: 3, workflowName: operation.workflowName, }, ctx, @@ -413,7 +224,6 @@ class FlueObserveBridge { startedAt: operation.startedAt, timestamp: new Date().toISOString(), type: "run_start", - v: 3, workflowName: operation.workflowName, }, ctx, @@ -472,7 +282,6 @@ class FlueObserveBridge { case "tool_start": this.handleToolStart(event); return; - case "tool_call": case "tool": this.handleToolCall(event); return; @@ -501,15 +310,12 @@ class FlueObserveBridge { return; } - const workflowName = - event.workflowName ?? - event.owner?.workflowName ?? - (typeof ctx?.id === "string" ? ctx.id : "unknown"); - const input = flueRunInput(event); + const workflowName = event.workflowName ?? ctx?.id ?? "unknown"; + const input = event.input; const metadata = { ...extractPayloadMetadata(input), ...extractEventMetadata(event, ctx), - ...(workflowName ? { "flue.workflow_name": workflowName } : {}), + "flue.workflow_name": workflowName, provider: "flue", }; const existing = this.runsById.get(event.runId); @@ -543,11 +349,10 @@ class FlueObserveBridge { return; } - const workflowName = - event.workflowName ?? (typeof ctx?.id === "string" ? ctx.id : "unknown"); + const workflowName = event.workflowName ?? ctx?.id ?? "unknown"; const metadata = { ...extractEventMetadata(event, ctx), - ...(workflowName ? { "flue.workflow_name": workflowName } : {}), + "flue.workflow_name": workflowName, "flue.workflow_phase": "resume", provider: "flue", }; @@ -664,20 +469,20 @@ class FlueObserveBridge { } private handleTurnRequest(event: FlueTurnRequestEvent): void { - const key = turnKey(event); + const key = event.turnId; if (!key) { return; } - const input = flueTurnRequestInput(event); + const input = event.request?.input; const operation = event.operationId ? this.operationsById.get(event.operationId) : undefined; const turnInput = prepareFlueTurnInput(event, input, operation); - const model = flueTurnRequestModel(event); - const provider = flueTurnRequestProvider(event); - const api = flueTurnRequestApi(event); - const reasoning = flueTurnRequestReasoning(event); + const model = event.request?.requestedModel; + const provider = event.request?.providerName ?? event.request?.providerId; + const api = event.request?.api; + const reasoning = event.request?.reasoningLevel; const metadata = { ...extractEventMetadata(event), ...(api ? { "flue.api": api } : {}), @@ -707,19 +512,20 @@ class FlueObserveBridge { } private handleTurn(event: FlueTurnEvent): void { - const key = turnKey(event); + const key = event.turnId; if (!key) { return; } const state = this.turnsByKey.get(key) ?? this.startSyntheticTurn(event); - const model = flueTurnModel(event); - const provider = flueTurnProvider(event); - const api = flueTurnApi(event); - const stopReason = flueTurnStopReason(event); - const usage = flueTurnUsage(event); - const output = flueTurnOutput(event); - const error = flueTurnError(event); + const model = + event.response?.responseModel ?? event.request?.requestedModel; + const provider = event.request?.providerName ?? event.request?.providerId; + const api = event.request?.api; + const stopReason = event.response?.finishReason; + const usage = event.response?.usage; + const output = event.response?.output; + const error = event.response?.error; const metadata = { ...state.metadata, ...extractEventMetadata(event), @@ -758,7 +564,7 @@ class FlueObserveBridge { return; } - const input = flueToolInput(event); + const input = event.args; const metadata = { ...extractEventMetadata(event), ...(event.toolName ? { "flue.tool_name": event.toolName } : {}), @@ -933,7 +739,7 @@ class FlueObserveBridge { } private parentSpanForEvent(event: FlueBaseEvent): Span | undefined { - const turn = turnKey(event); + const turn = event.turnId; if (turn) { const turnState = this.turnsByKey.get(turn); if (turnState) { @@ -1017,9 +823,10 @@ class FlueObserveBridge { } private startSyntheticTurn(event: FlueTurnEvent): SpanState { - const model = flueTurnModel(event); - const provider = flueTurnProvider(event); - const api = flueTurnApi(event); + const model = + event.response?.responseModel ?? event.request?.requestedModel; + const provider = event.request?.providerName ?? event.request?.providerId; + const api = event.request?.api; const metadata = { ...extractEventMetadata(event), ...(api ? { "flue.api": api } : {}), @@ -1208,10 +1015,7 @@ function extractEventMetadata( ...(event.taskId ? { "flue.task_id": event.taskId } : {}), ...(event.operationId ? { "flue.operation_id": event.operationId } : {}), ...(event.turnId ? { "flue.turn_id": event.turnId } : {}), - ...(typeof ctx?.id === "string" ? { "flue.context_id": ctx.id } : {}), - ...(typeof ctx?.runId === "string" - ? { "flue.context_run_id": ctx.runId } - : {}), + ...(ctx ? { "flue.context_id": ctx.id } : {}), }; } @@ -1226,19 +1030,9 @@ function extractPayloadMetadata(payload: unknown): Record { return Object.fromEntries(Object.entries(metadata)); } -function flueRunInput(event: FlueRunStartEvent): unknown { - return event.input !== undefined ? event.input : event.payload; -} - -function flueTurnRequestInput( - event: FlueTurnRequestEvent, -): FlueTurnRequestEvent["input"] { - return event.request?.input ?? event.input; -} - function prepareFlueTurnInput( event: FlueTurnRequestEvent, - input: FlueTurnRequestEvent["input"], + input: NonNullable["input"], operation: SpanState | undefined, ): { messages: unknown[] | undefined; metadata: Record } { const messages = input?.messages; @@ -1347,90 +1141,15 @@ function flueOperationInput(event: FlueOperationEvent): unknown { : undefined; } -function flueTurnRequestModel(event: FlueTurnRequestEvent): string | undefined { - return event.request?.requestedModel ?? event.request?.model ?? event.model; -} - -function flueTurnRequestProvider( - event: FlueTurnRequestEvent, -): string | undefined { - return ( - event.request?.providerName ?? event.provider ?? event.request?.providerId - ); -} - -function flueTurnRequestApi(event: FlueTurnRequestEvent): string | undefined { - return event.request?.api ?? event.api; -} - -function flueTurnRequestReasoning( - event: FlueTurnRequestEvent, -): string | undefined { - return ( - event.request?.reasoningLevel ?? event.request?.reasoning ?? event.reasoning - ); -} - -function flueTurnModel(event: FlueTurnEvent): string | undefined { - return ( - event.response?.responseModel ?? - event.request?.requestedModel ?? - event.request?.model ?? - event.model - ); -} - -function flueTurnProvider(event: FlueTurnEvent): string | undefined { - return ( - event.request?.providerName ?? event.provider ?? event.request?.providerId - ); -} - -function flueTurnApi(event: FlueTurnEvent): string | undefined { - return event.request?.api ?? event.api; -} - -function flueTurnUsage(event: FlueTurnEvent): unknown { - return event.response?.usage ?? event.usage; -} - -function flueTurnOutput(event: FlueTurnEvent): unknown { - return event.response?.output ?? event.output; -} - -function flueTurnStopReason(event: FlueTurnEvent): string | undefined { - return ( - event.response?.finishReason ?? - event.response?.stopReason ?? - event.stopReason - ); -} - -function flueTurnError(event: FlueTurnEvent): unknown { - return ( - event.response?.error ?? event.response?.errorInfo?.message ?? event.error - ); -} - -function flueToolInput(event: FlueToolStartEvent): unknown { - if (event.args !== undefined) { - return event.args; - } - if (event.arguments !== undefined) { - return event.arguments; - } - return event.input; -} - function flueToolOutput(event: FlueToolCallEvent): unknown { if (Object.hasOwn(event, "effectiveResult")) { return event.effectiveResult; } - return event.output !== undefined ? event.output : event.result; + return event.result; } function flueToolError(event: FlueToolCallEvent): unknown { - return event.error ?? event.errorInfo?.message ?? flueToolOutput(event); + return event.errorInfo?.message ?? flueToolOutput(event); } function operationOutput(event: FlueOperationEvent): unknown { @@ -1529,10 +1248,6 @@ function eventTime(value: unknown): number | undefined { return Number.isFinite(timestamp) ? timestamp / 1000 : undefined; } -function turnKey(event: FlueBaseEvent): string | undefined { - return event.turnId; -} - function toolKey(event: FlueBaseEvent & { toolCallId?: string }): string { return `${event.turnId ?? event.operationId ?? event.taskId ?? event.runId ?? "unknown"}:${event.toolCallId ?? "unknown"}`; } @@ -1594,11 +1309,7 @@ function runWithCurrentSpanStore( const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const currentSpanStore = contextManager - ? ( - contextManager as { - [BRAINTRUST_CURRENT_SPAN_STORE]?: CurrentSpanStore; - } - )[BRAINTRUST_CURRENT_SPAN_STORE] + ? Reflect.get(contextManager, BRAINTRUST_CURRENT_SPAN_STORE) : undefined; if (contextManager && typeof currentSpanStore?.run === "function") { diff --git a/js/src/instrumentation/plugins/genkit-channels.ts b/js/src/instrumentation/providers/genkit-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/genkit-channels.ts rename to js/src/instrumentation/providers/genkit-channels.ts diff --git a/js/src/instrumentation/plugins/genkit-plugin.test.ts b/js/src/instrumentation/providers/genkit-instrumentation.test.ts similarity index 91% rename from js/src/instrumentation/plugins/genkit-plugin.test.ts rename to js/src/instrumentation/providers/genkit-instrumentation.test.ts index dcf41c0f7..11f172e5b 100644 --- a/js/src/instrumentation/plugins/genkit-plugin.test.ts +++ b/js/src/instrumentation/providers/genkit-instrumentation.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { _exportsForTestingOnly, initLogger } from "../../logger"; -import { GenkitPlugin } from "./genkit-plugin"; +import { registerGenkitInstrumentation } from "./genkit-instrumentation"; import { genkitChannels } from "./genkit-channels"; function singleQueueStream( @@ -35,9 +35,7 @@ async function collectAsync(stream: AsyncIterable): Promise { return chunks; } -describe("GenkitPlugin stream patching", () => { - const plugin = new GenkitPlugin(); - +describe("registerGenkitInstrumentation stream patching", () => { beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); }); @@ -45,18 +43,17 @@ describe("GenkitPlugin stream patching", () => { beforeEach(() => { _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "genkit-plugin.test.ts", + projectName: "genkit-instrumentation.test.ts", projectId: "test-project-id", }); }); afterEach(() => { - plugin.disable(); _exportsForTestingOnly.clearTestBackgroundLogger(); }); it("does not consume generateStream chunks before user code reads them", async () => { - plugin.enable(); + registerGenkitInstrumentation(); const stream = singleQueueStream([{ text: "hello" }, { text: " world" }]); const result = genkitChannels.generateStream.traceSync( @@ -85,7 +82,7 @@ describe("GenkitPlugin stream patching", () => { }); it("does not consume action.stream chunks before user code reads them", async () => { - plugin.enable(); + registerGenkitInstrumentation(); const stream = singleQueueStream(["first", "second"]); const action = Object.assign(() => Promise.resolve(), { __action: { diff --git a/js/src/instrumentation/plugins/genkit-plugin.ts b/js/src/instrumentation/providers/genkit-instrumentation.ts similarity index 85% rename from js/src/instrumentation/plugins/genkit-plugin.ts rename to js/src/instrumentation/providers/genkit-instrumentation.ts index 90edb66e4..e485295ea 100644 --- a/js/src/instrumentation/plugins/genkit-plugin.ts +++ b/js/src/instrumentation/providers/genkit-instrumentation.ts @@ -1,8 +1,7 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { traceAsyncChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { ChannelMessage } from "../core/channel-definitions"; @@ -38,61 +37,49 @@ type SpanState = { startTime: number; }; -export class GenkitPlugin extends BasePlugin { - protected onEnable(): void { +class GenkitInstrumentationConsumer { + public register(): void { this.subscribeToGenkitChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToGenkitChannels(): void { - this.unsubscribers.push( - traceAsyncChannel(genkitChannels.generate, { - name: "genkit.generate", - type: SpanTypeAttribute.LLM, - extractInput: ([input]) => extractGenerateInput(input), - extractOutput: extractGenerateOutput, - extractMetadata: (result, event) => - extractGenerateResponseMetadata(result, event?.arguments?.[0]), - extractMetrics: (result) => parseGenkitUsageMetrics(result?.usage), - }), - ); + traceAsyncChannel(genkitChannels.generate, { + name: "genkit.generate", + type: SpanTypeAttribute.LLM, + extractInput: ([input]) => extractGenerateInput(input), + extractOutput: extractGenerateOutput, + extractMetadata: (result, event) => + extractGenerateResponseMetadata(result, event?.arguments?.[0]), + extractMetrics: (result) => parseGenkitUsageMetrics(result?.usage), + }); - this.unsubscribers.push( - traceSyncStreamChannel(genkitChannels.generateStream, { - name: "genkit.generateStream", - type: SpanTypeAttribute.LLM, - extractInput: ([input]) => extractGenerateInput(input), - patchResult: ({ result, span, startTime }) => - patchGenerateStreamResult(result, span, startTime), - }), - ); + traceSyncStreamChannel(genkitChannels.generateStream, { + name: "genkit.generateStream", + type: SpanTypeAttribute.LLM, + extractInput: ([input]) => extractGenerateInput(input), + patchResult: ({ result, span, startTime }) => + patchGenerateStreamResult(result, span, startTime), + }); - this.unsubscribers.push( - traceAsyncChannel(genkitChannels.embed, { - name: "genkit.embed", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params]) => extractEmbedInput(params), - extractOutput: (result) => summarizeEmbeddingResult(result), - extractMetadata: (_result, event) => - extractEmbedMetadata(event?.arguments?.[0]), - extractMetrics: () => ({}), - }), - ); + traceAsyncChannel(genkitChannels.embed, { + name: "genkit.embed", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params]) => extractEmbedInput(params), + extractOutput: (result) => summarizeEmbeddingResult(result), + extractMetadata: (_result, event) => + extractEmbedMetadata(event?.arguments?.[0]), + extractMetrics: () => ({}), + }); - this.unsubscribers.push( - traceAsyncChannel(genkitChannels.embedMany, { - name: "genkit.embedMany", - type: SpanTypeAttribute.FUNCTION, - extractInput: ([params]) => extractEmbedManyInput(params), - extractOutput: summarizeEmbeddingResult, - extractMetadata: (_result, event) => - extractEmbedMetadata(event?.arguments?.[0]), - extractMetrics: () => ({}), - }), - ); + traceAsyncChannel(genkitChannels.embedMany, { + name: "genkit.embedMany", + type: SpanTypeAttribute.FUNCTION, + extractInput: ([params]) => extractEmbedManyInput(params), + extractOutput: summarizeEmbeddingResult, + extractMetadata: (_result, event) => + extractEmbedMetadata(event?.arguments?.[0]), + extractMetrics: () => ({}), + }); this.subscribeToActionRun(); this.subscribeToActionSpan(); @@ -105,10 +92,8 @@ export class GenkitPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindActionCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => startActionRunSpan(event), + bindActionCurrentSpanStoreToStart(tracingChannel, states, (event) => + startActionRunSpan(event), ); const handlers: IsoChannelHandlers< @@ -147,10 +132,6 @@ export class GenkitPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToActionSpan(): void { @@ -159,10 +140,8 @@ export class GenkitPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindActionCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => startActionSpan(event), + bindActionCurrentSpanStoreToStart(tracingChannel, states, (event) => + startActionSpan(event), ); const handlers: IsoChannelHandlers< @@ -202,25 +181,19 @@ export class GenkitPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToActionStream(): void { - this.unsubscribers.push( - traceSyncStreamChannel(genkitChannels.actionStream, { - name: "genkit.action.stream", - type: SpanTypeAttribute.TASK, - extractInput: ([input], event) => ({ - input, - metadata: actionMetadataForLog(extractActionMetadata(event.self)), - }), - patchResult: ({ result, span, startTime }) => - patchActionStreamResult(result, span, startTime), + traceSyncStreamChannel(genkitChannels.actionStream, { + name: "genkit.action.stream", + type: SpanTypeAttribute.TASK, + extractInput: ([input], event) => ({ + input, + metadata: actionMetadataForLog(extractActionMetadata(event.self)), }), - ); + patchResult: ({ result, span, startTime }) => + patchActionStreamResult(result, span, startTime), + }); } } @@ -307,7 +280,7 @@ function bindActionCurrentSpanStoreToStart< tracingChannel: IsoTracingChannel>, states: WeakMap, create: (event: ChannelMessage) => SpanState | undefined, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const startChannel = tracingChannel.start as @@ -316,7 +289,6 @@ function bindActionCurrentSpanStoreToStart< store: CurrentSpanStore, callback: (event: ChannelMessage) => unknown, ) => void; - unbindStore?: (store: CurrentSpanStore) => void; } & object) | undefined; const currentSpanStore = contextManager @@ -328,7 +300,7 @@ function bindActionCurrentSpanStoreToStart< : undefined; if (!startChannel?.bindStore || !currentSpanStore) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -339,10 +311,6 @@ function bindActionCurrentSpanStoreToStart< ? contextManager!.wrapSpanForStore(state.span) : currentSpanStore.getStore(); }); - - return () => { - startChannel.unbindStore?.(currentSpanStore); - }; } function normalizeInput(input: GenkitGenerateInput): GenkitGenerateInput { @@ -771,3 +739,10 @@ function pickNumberMetrics( function stringValue(value: unknown): string | undefined { return typeof value === "string" ? value : undefined; } + +let genkitInstrumentationConsumer: GenkitInstrumentationConsumer | undefined; + +export function registerGenkitInstrumentation(): void { + genkitInstrumentationConsumer ??= new GenkitInstrumentationConsumer(); + genkitInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/github-copilot-channels.ts b/js/src/instrumentation/providers/github-copilot-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/github-copilot-channels.ts rename to js/src/instrumentation/providers/github-copilot-channels.ts diff --git a/js/src/instrumentation/plugins/github-copilot-plugin.test.ts b/js/src/instrumentation/providers/github-copilot-instrumentation.test.ts similarity index 79% rename from js/src/instrumentation/plugins/github-copilot-plugin.test.ts rename to js/src/instrumentation/providers/github-copilot-instrumentation.test.ts index 17ecb38d8..5e88b3b51 100644 --- a/js/src/instrumentation/plugins/github-copilot-plugin.test.ts +++ b/js/src/instrumentation/providers/github-copilot-instrumentation.test.ts @@ -1,8 +1,8 @@ import { describe, expect, it } from "vitest"; import { - GitHubCopilotPlugin, + registerGitHubCopilotInstrumentation, extractMetricsFromUsage, -} from "./github-copilot-plugin"; +} from "./github-copilot-instrumentation"; describe("extractMetricsFromUsage", () => { it("maps input/output tokens to standard metric keys", () => { @@ -119,38 +119,8 @@ describe("extractMetricsFromUsage", () => { }); }); -describe("GitHubCopilotPlugin lifecycle", () => { - it("enables without throwing", () => { - const plugin = new GitHubCopilotPlugin(); - expect(() => plugin.enable()).not.toThrow(); - plugin.disable(); - }); - - it("disables without throwing", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("enable is idempotent", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - expect(() => plugin.enable()).not.toThrow(); - plugin.disable(); - }); - - it("disable is idempotent", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - plugin.disable(); - expect(() => plugin.disable()).not.toThrow(); - }); - - it("can be re-enabled after disable", () => { - const plugin = new GitHubCopilotPlugin(); - plugin.enable(); - plugin.disable(); - expect(() => plugin.enable()).not.toThrow(); - plugin.disable(); +describe("registerGitHubCopilotInstrumentation", () => { + it("registers without throwing", () => { + expect(() => registerGitHubCopilotInstrumentation()).not.toThrow(); }); }); diff --git a/js/src/instrumentation/plugins/github-copilot-plugin.ts b/js/src/instrumentation/providers/github-copilot-instrumentation.ts similarity index 97% rename from js/src/instrumentation/plugins/github-copilot-plugin.ts rename to js/src/instrumentation/providers/github-copilot-instrumentation.ts index 8fa156cfc..b5f14976f 100644 --- a/js/src/instrumentation/plugins/github-copilot-plugin.ts +++ b/js/src/instrumentation/providers/github-copilot-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { IsoChannelHandlers } from "../../isomorph"; import { startSpan as startBaseSpan } from "../../logger"; import type { Span } from "../../logger"; @@ -672,7 +671,7 @@ function isGitHubCopilotSession(value: unknown): value is GitHubCopilotSession { } // --------------------------------------------------------------------------- -// Plugin and handler factory +// Consumer and handler factory // --------------------------------------------------------------------------- // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -756,18 +755,11 @@ function makeSessionHandlers( }; } -export class GitHubCopilotPlugin extends BasePlugin { - protected onEnable(): void { +class GitHubCopilotInstrumentationConsumer { + public register(): void { this.subscribeToSessionChannels(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToSessionChannels(): void { const createChannel = gitHubCopilotChannels.createSession.tracingChannel(); const resumeChannel = gitHubCopilotChannels.resumeSession.tracingChannel(); @@ -787,10 +779,15 @@ export class GitHubCopilotPlugin extends BasePlugin { createChannel.subscribe(createHandlers); resumeChannel.subscribe(resumeHandlers); - - this.unsubscribers.push( - () => createChannel.unsubscribe(createHandlers), - () => resumeChannel.unsubscribe(resumeHandlers), - ); } } + +let gitHubCopilotInstrumentationConsumer: + | GitHubCopilotInstrumentationConsumer + | undefined; + +export function registerGitHubCopilotInstrumentation(): void { + gitHubCopilotInstrumentationConsumer ??= + new GitHubCopilotInstrumentationConsumer(); + gitHubCopilotInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/google-adk-channels.ts b/js/src/instrumentation/providers/google-adk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/google-adk-channels.ts rename to js/src/instrumentation/providers/google-adk-channels.ts diff --git a/js/src/instrumentation/plugins/google-adk-plugin.test.ts b/js/src/instrumentation/providers/google-adk-instrumentation.test.ts similarity index 91% rename from js/src/instrumentation/plugins/google-adk-plugin.test.ts rename to js/src/instrumentation/providers/google-adk-instrumentation.test.ts index 073af75f4..f4efcfc7d 100644 --- a/js/src/instrumentation/plugins/google-adk-plugin.test.ts +++ b/js/src/instrumentation/providers/google-adk-instrumentation.test.ts @@ -15,7 +15,7 @@ vi.mock("../../isomorph", () => ({ }, })); -import { GoogleADKPlugin } from "./google-adk-plugin"; +import { registerGoogleADKInstrumentation } from "./google-adk-instrumentation"; import iso from "../../isomorph"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; @@ -44,26 +44,19 @@ vi.mock("../../logger", () => ({ }, })); -describe("GoogleADKPlugin", () => { - let plugin: GoogleADKPlugin; +describe("registerGoogleADKInstrumentation", () => { let mockChannel: any; let subscribeSpy: any; - let unsubscribeSpy: any; let bindStoreSpy: any; - let unbindStoreSpy: any; beforeEach(() => { subscribeSpy = vi.fn(); - unsubscribeSpy = vi.fn(); bindStoreSpy = vi.fn(); - unbindStoreSpy = vi.fn(); mockChannel = { subscribe: subscribeSpy, - unsubscribe: unsubscribeSpy, hasSubscribers: false, start: { bindStore: bindStoreSpy, - unbindStore: unbindStoreSpy, }, }; @@ -71,16 +64,15 @@ describe("GoogleADKPlugin", () => { mockStartSpan.mockClear(); mockInternalGetGlobalState.mockReset(); mockInternalGetGlobalState.mockReturnValue(undefined); - plugin = new GoogleADKPlugin(); }); afterEach(() => { vi.clearAllMocks(); }); - describe("enable/disable lifecycle", () => { - it("should subscribe to channels when enabled", () => { - plugin.enable(); + describe("registration", () => { + it("subscribes to channels", () => { + registerGoogleADKInstrumentation(); // Should subscribe to 3 channels: runner.runAsync, agent.runAsync, tool.runAsync expect(mockNewTracingChannel).toHaveBeenCalledWith( @@ -94,43 +86,11 @@ describe("GoogleADKPlugin", () => { ); expect(subscribeSpy).toHaveBeenCalledTimes(3); }); - - it("should not subscribe multiple times if enabled twice", () => { - plugin.enable(); - const firstCallCount = subscribeSpy.mock.calls.length; - - plugin.enable(); - const secondCallCount = subscribeSpy.mock.calls.length; - - expect(firstCallCount).toBe(secondCallCount); - }); - - it("should unsubscribe from channels when disabled", () => { - plugin.enable(); - plugin.disable(); - - expect(unsubscribeSpy).toHaveBeenCalled(); - }); - - it("should clear unsubscribers array after disable", () => { - plugin.enable(); - plugin.disable(); - - // Enable again should re-subscribe - subscribeSpy.mockClear(); - plugin.enable(); - - expect(subscribeSpy).toHaveBeenCalledTimes(3); - }); - - it("should not crash when disabled without being enabled", () => { - expect(() => plugin.disable()).not.toThrow(); - }); }); describe("runner.runAsync channel", () => { it("should create a TASK span with runner metadata on start", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); // Find the first subscribe call (runner channel) const handlers = subscribeSpy.mock.calls[0][0]; @@ -163,7 +123,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle stream end with async iterable result", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -196,7 +156,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle error events", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -228,7 +188,7 @@ describe("GoogleADKPlugin", () => { }, } as any); - plugin.enable(); + registerGoogleADKInstrumentation(); expect(bindStoreSpy).toHaveBeenNthCalledWith( 1, @@ -367,7 +327,7 @@ describe("GoogleADKPlugin", () => { ])( "normalizes $name", async ({ usageMetadata, expectedMetrics, absentMetrics }) => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -398,7 +358,7 @@ describe("GoogleADKPlugin", () => { ); it("preserves explicitly reported zero usage", async () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -447,7 +407,7 @@ describe("GoogleADKPlugin", () => { describe("agent.runAsync channel", () => { it("should create a TASK span with agent metadata on start", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); // Agent channel is the second subscribe call const handlers = subscribeSpy.mock.calls[1][0]; @@ -474,7 +434,7 @@ describe("GoogleADKPlugin", () => { }); it("uses the invoked agent instance for names when parent context still points at a parent agent", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[1][0]; const event = { @@ -511,7 +471,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle agent without a name gracefully", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[1][0]; const event = { @@ -537,7 +497,7 @@ describe("GoogleADKPlugin", () => { }, } as any); - plugin.enable(); + registerGoogleADKInstrumentation(); expect(bindStoreSpy).toHaveBeenNthCalledWith( 2, @@ -588,7 +548,7 @@ describe("GoogleADKPlugin", () => { .mockImplementationOnce(() => runnerSpan) .mockImplementationOnce(() => agentSpan); - plugin.enable(); + registerGoogleADKInstrumentation(); const runnerHandlers = subscribeSpy.mock.calls[0][0]; const agentHandlers = subscribeSpy.mock.calls[1][0]; @@ -638,7 +598,7 @@ describe("GoogleADKPlugin", () => { describe("tool.runAsync channel", () => { it("should create a TOOL span on start", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); // Tool channel is the third subscribe call const handlers = subscribeSpy.mock.calls[2][0]; @@ -675,7 +635,7 @@ describe("GoogleADKPlugin", () => { }); it("should log output and metrics on asyncEnd", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[2][0]; const event: any = { @@ -706,7 +666,7 @@ describe("GoogleADKPlugin", () => { }); it("should handle tool execution errors", () => { - plugin.enable(); + registerGoogleADKInstrumentation(); const handlers = subscribeSpy.mock.calls[2][0]; const event: any = { diff --git a/js/src/instrumentation/plugins/google-adk-plugin.ts b/js/src/instrumentation/providers/google-adk-instrumentation.ts similarity index 95% rename from js/src/instrumentation/plugins/google-adk-plugin.ts rename to js/src/instrumentation/providers/google-adk-instrumentation.ts index 623264016..994779305 100644 --- a/js/src/instrumentation/plugins/google-adk-plugin.ts +++ b/js/src/instrumentation/providers/google-adk-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; @@ -51,9 +50,9 @@ type GoogleADKStreamChannel = | typeof googleADKChannels.agentRunAsync; /** - * Auto-instrumentation plugin for the Google ADK. + * Internal auto-instrumentation consumer for the Google ADK. * - * This plugin subscribes to orchestrion channels for Google ADK methods + * This consumer subscribes to orchestrion channels for Google ADK methods * and creates Braintrust spans to track: * - Runner.runAsync — top-level agent execution (TASK span) * - BaseAgent.runAsync — individual agent invocations (TASK span) @@ -62,25 +61,16 @@ type GoogleADKStreamChannel = * LLM calls made through ADK are automatically captured by the existing * @google/genai instrumentation since ADK uses GenAI internally. */ -export class GoogleADKPlugin extends BasePlugin { +class GoogleADKInstrumentationConsumer { private activeRunnerSpans = new Map(); private activeAgentSpans = new Map(); - protected onEnable(): void { + public register(): void { this.subscribeToRunnerRunAsync(); this.subscribeToAgentRunAsync(); this.subscribeToToolRunAsync(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - this.activeRunnerSpans.clear(); - this.activeAgentSpans.clear(); - } - private subscribeToRunnerRunAsync(): void { const tracingChannel = googleADKChannels.runnerRunAsync.tracingChannel() as IsoTracingChannel< @@ -126,11 +116,7 @@ export class GoogleADKPlugin extends BasePlugin { return { span, startTime, events: [], contextKey }; }; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - createState, - ); + bindCurrentSpanStoreToStart(tracingChannel, states, createState); const handlers: IsoChannelHandlers< ChannelMessage @@ -189,10 +175,6 @@ export class GoogleADKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToAgentRunAsync(): void { @@ -262,11 +244,7 @@ export class GoogleADKPlugin extends BasePlugin { return { span, startTime, events: [], contextKey, name: agentName }; }; - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - createState, - ); + bindCurrentSpanStoreToStart(tracingChannel, states, createState); const handlers: IsoChannelHandlers< ChannelMessage @@ -324,10 +302,6 @@ export class GoogleADKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToToolRunAsync(): void { @@ -413,9 +387,6 @@ export class GoogleADKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } } @@ -536,7 +507,7 @@ function bindCurrentSpanStoreToStart< tracingChannel: IsoTracingChannel>, states: WeakMap, create: (event: ChannelMessage) => TState, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const startChannel = tracingChannel.start as @@ -545,7 +516,6 @@ function bindCurrentSpanStoreToStart< store: CurrentSpanStore, callback: (event: ChannelMessage) => unknown, ) => void; - unbindStore?: (store: CurrentSpanStore) => void; } & object) | undefined; const currentSpanStore = contextManager @@ -557,7 +527,7 @@ function bindCurrentSpanStoreToStart< : undefined; if (!startChannel?.bindStore || !currentSpanStore) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -566,10 +536,6 @@ function bindCurrentSpanStoreToStart< ).span; return contextManager.wrapSpanForStore(span); }); - - return () => { - startChannel.unbindStore?.(currentSpanStore); - }; } // ---- Helper functions ---- @@ -1026,3 +992,12 @@ function cleanMetrics(metrics: Record): Record { } return cleaned; } + +let googleADKInstrumentationConsumer: + | GoogleADKInstrumentationConsumer + | undefined; + +export function registerGoogleADKInstrumentation(): void { + googleADKInstrumentationConsumer ??= new GoogleADKInstrumentationConsumer(); + googleADKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/google-genai-channels.ts b/js/src/instrumentation/providers/google-genai-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/google-genai-channels.ts rename to js/src/instrumentation/providers/google-genai-channels.ts diff --git a/js/src/instrumentation/plugins/google-genai-plugin.test.ts b/js/src/instrumentation/providers/google-genai-instrumentation.test.ts similarity index 95% rename from js/src/instrumentation/plugins/google-genai-plugin.test.ts rename to js/src/instrumentation/providers/google-genai-instrumentation.test.ts index f14690e73..175cfb309 100644 --- a/js/src/instrumentation/plugins/google-genai-plugin.test.ts +++ b/js/src/instrumentation/providers/google-genai-instrumentation.test.ts @@ -25,7 +25,7 @@ vi.mock("../../isomorph", () => ({ }, })); -import { GoogleGenAIPlugin } from "./google-genai-plugin"; +import { registerGoogleGenAIInstrumentation } from "./google-genai-instrumentation"; import { startSpan } from "../../logger"; import iso from "../../isomorph"; @@ -50,66 +50,27 @@ vi.mock("../../logger", () => ({ }, })); -describe("GoogleGenAIPlugin", () => { - let plugin: GoogleGenAIPlugin; +describe("registerGoogleGenAIInstrumentation", () => { let mockChannel: any; let subscribeSpy: any; - let unsubscribeSpy: any; beforeEach(() => { subscribeSpy = vi.fn(); - unsubscribeSpy = vi.fn(); mockChannel = { subscribe: subscribeSpy, - unsubscribe: unsubscribeSpy, hasSubscribers: false, }; mockNewTracingChannel.mockReturnValue(mockChannel); - plugin = new GoogleGenAIPlugin(); }); afterEach(() => { vi.clearAllMocks(); }); - describe("enable/disable lifecycle", () => { - it("should not subscribe multiple times if enabled twice", () => { - plugin.enable(); - const firstCallCount = subscribeSpy.mock.calls.length; - - plugin.enable(); - const secondCallCount = subscribeSpy.mock.calls.length; - - expect(firstCallCount).toBe(secondCallCount); - }); - - it("should unsubscribe from channels when disabled", () => { - plugin.enable(); - plugin.disable(); - - expect(unsubscribeSpy).toHaveBeenCalled(); - }); - - it("should clear unsubscribers array after disable", () => { - plugin.enable(); - plugin.disable(); - - // Enable again should re-subscribe - subscribeSpy.mockClear(); - plugin.enable(); - - expect(subscribeSpy).toHaveBeenCalled(); - }); - - it("should not crash when disabled without being enabled", () => { - expect(() => plugin.disable()).not.toThrow(); - }); - }); - describe("generateContent channel subscription", () => { it("should extract input correctly", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const subscribeCall = subscribeSpy.mock.calls.find( (call: any) => @@ -199,7 +160,7 @@ describe("GoogleGenAIPlugin", () => { ])( "normalizes $name", ({ usageMetadata, expectedMetrics, absentMetrics }) => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -227,7 +188,7 @@ describe("GoogleGenAIPlugin", () => { ); it("preserves explicitly reported zero usage", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[0][0]; const event: any = { @@ -275,7 +236,7 @@ describe("GoogleGenAIPlugin", () => { describe("interactions.create channel subscription", () => { it("subscribes to the interactions.create channel", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); expect(mockNewTracingChannel).toHaveBeenCalledWith( "orchestrion:@google/genai:interactions.create", @@ -284,7 +245,7 @@ describe("GoogleGenAIPlugin", () => { }); it("logs non-streaming interaction output and metrics", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[3][0]; const scheduledAt = new Date("2026-01-02T03:04:05.000Z"); @@ -412,7 +373,7 @@ describe("GoogleGenAIPlugin", () => { }); it("preserves zero and missing interaction usage values", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[3][0]; const event: any = { @@ -488,7 +449,7 @@ describe("GoogleGenAIPlugin", () => { }); it("does not trace background interaction tasks", () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const handlers = subscribeSpy.mock.calls[3][0]; const event: any = { @@ -512,7 +473,7 @@ describe("GoogleGenAIPlugin", () => { }); it("aggregates streaming interaction events when consumed", async () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); async function* stream() { yield { @@ -599,7 +560,7 @@ describe("GoogleGenAIPlugin", () => { }); it("ends the interaction span when a stream errors", async () => { - plugin.enable(); + registerGoogleGenAIInstrumentation(); const streamError = new Error("stream failed"); async function* stream() { @@ -651,7 +612,7 @@ describe("Google GenAI serialization functions", () => { contents: "Hello world", }; - // Since these are private functions, we'll test them through the plugin behavior + // Since these are private functions, test them through consumer behavior. // For now, we'll verify the structure by examining what gets logged expect(params.model).toBe("gemini-pro"); expect(params.contents).toBe("Hello world"); diff --git a/js/src/instrumentation/plugins/google-genai-plugin.ts b/js/src/instrumentation/providers/google-genai-instrumentation.ts similarity index 92% rename from js/src/instrumentation/plugins/google-genai-plugin.ts rename to js/src/instrumentation/providers/google-genai-instrumentation.ts index fc70e112c..85e5ae049 100644 --- a/js/src/instrumentation/plugins/google-genai-plugin.ts +++ b/js/src/instrumentation/providers/google-genai-instrumentation.ts @@ -1,5 +1,4 @@ -import { BasePlugin } from "../core"; -import { traceStreamingChannel, unsubscribeAll } from "../core/channel-tracing"; +import { traceStreamingChannel } from "../core/channel-tracing"; import type { ChannelMessage, ErrorOf, @@ -74,29 +73,25 @@ function createWrapperParityEvent(args: { } /** - * Auto-instrumentation plugin for the Google GenAI SDK. + * Internal auto-instrumentation consumer for the Google GenAI SDK. * - * This plugin subscribes to orchestrion channels for Google GenAI SDK methods + * This consumer subscribes to orchestrion channels for Google GenAI SDK methods * and creates Braintrust spans to track: * - models.generateContent (non-streaming) * - models.generateContentStream (streaming) * - models.embedContent (embeddings) * - * The plugin handles: + * The consumer handles: * - Google-specific token metrics (promptTokenCount, candidatesTokenCount, cachedContentTokenCount) * - Processing streaming responses * - Converting inline data (images) to Attachment objects * - Tool calls (functionCall, functionResponse) and executable code results */ -export class GoogleGenAIPlugin extends BasePlugin { - protected onEnable(): void { +class GoogleGenAIInstrumentationConsumer { + public register(): void { this.subscribeToGoogleGenAIChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToGoogleGenAIChannels(): void { this.subscribeToGenerateContentChannel(); this.subscribeToGenerateContentStreamChannel(); @@ -110,32 +105,28 @@ export class GoogleGenAIPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => { - const params = event.arguments[0]; - const input = serializeGenerateContentInput(params); - const metadata = extractGenerateContentMetadata(params); - const span = startBaseSpan( - withSpanInstrumentationName( - { - name: "generate_content", - spanAttributes: { - type: SpanTypeAttribute.LLM, - }, - event: createWrapperParityEvent({ input, metadata }), + bindCurrentSpanStoreToStart(tracingChannel, states, (event) => { + const params = event.arguments[0]; + const input = serializeGenerateContentInput(params); + const metadata = extractGenerateContentMetadata(params); + const span = startBaseSpan( + withSpanInstrumentationName( + { + name: "generate_content", + spanAttributes: { + type: SpanTypeAttribute.LLM, }, - INSTRUMENTATION_NAMES.GOOGLE_GENAI, - ), - ); - - return { - span, - startTime: getCurrentUnixTimestamp(), - }; - }, - ); + event: createWrapperParityEvent({ input, metadata }), + }, + INSTRUMENTATION_NAMES.GOOGLE_GENAI, + ), + ); + + return { + span, + startTime: getCurrentUnixTimestamp(), + }; + }); const handlers: IsoChannelHandlers> = { @@ -192,10 +183,6 @@ export class GoogleGenAIPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToGenerateContentStreamChannel(): void { @@ -228,9 +215,6 @@ export class GoogleGenAIPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private subscribeToEmbedContentChannel(): void { @@ -239,32 +223,28 @@ export class GoogleGenAIPlugin extends BasePlugin { ChannelMessage >; const states = new WeakMap(); - const unbindCurrentSpanStore = bindCurrentSpanStoreToStart( - tracingChannel, - states, - (event) => { - const params = event.arguments[0]; - const input = serializeEmbedContentInput(params); - const metadata = extractEmbedContentMetadata(params); - const span = startBaseSpan( - withSpanInstrumentationName( - { - name: "embed_content", - spanAttributes: { - type: SpanTypeAttribute.LLM, - }, - event: createWrapperParityEvent({ input, metadata }), + bindCurrentSpanStoreToStart(tracingChannel, states, (event) => { + const params = event.arguments[0]; + const input = serializeEmbedContentInput(params); + const metadata = extractEmbedContentMetadata(params); + const span = startBaseSpan( + withSpanInstrumentationName( + { + name: "embed_content", + spanAttributes: { + type: SpanTypeAttribute.LLM, }, - INSTRUMENTATION_NAMES.GOOGLE_GENAI, - ), - ); - - return { - span, - startTime: getCurrentUnixTimestamp(), - }; - }, - ); + event: createWrapperParityEvent({ input, metadata }), + }, + INSTRUMENTATION_NAMES.GOOGLE_GENAI, + ), + ); + + return { + span, + startTime: getCurrentUnixTimestamp(), + }; + }); const handlers: IsoChannelHandlers> = { start: (event) => { @@ -316,33 +296,26 @@ export class GoogleGenAIPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindCurrentSpanStore?.(); - tracingChannel.unsubscribe(handlers); - }); } private subscribeToInteractionsCreateChannel(): void { - this.unsubscribers.push( - traceStreamingChannel( - googleGenAIChannels.interactionsCreate as InteractionsCreateChannel, - { - name: "create_interaction", - shouldTrace: ([params]) => !isBackgroundInteractionCreate(params), - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => ({ - input: serializeInteractionInput(params), - metadata: extractInteractionMetadata(params), - }), - extractOutput: (result) => serializeInteractionValue(result), - extractMetadata: (result) => - extractInteractionResponseMetadata(result), - extractMetrics: (result, startTime) => - cleanMetrics(extractInteractionMetrics(result, startTime)), - aggregateChunks: (chunks, _result, _event, startTime) => - aggregateInteractionEvents(chunks, startTime), - }, - ), + traceStreamingChannel( + googleGenAIChannels.interactionsCreate as InteractionsCreateChannel, + { + name: "create_interaction", + shouldTrace: ([params]) => !isBackgroundInteractionCreate(params), + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => ({ + input: serializeInteractionInput(params), + metadata: extractInteractionMetadata(params), + }), + extractOutput: (result) => serializeInteractionValue(result), + extractMetadata: (result) => extractInteractionResponseMetadata(result), + extractMetrics: (result, startTime) => + cleanMetrics(extractInteractionMetrics(result, startTime)), + aggregateChunks: (chunks, _result, _event, startTime) => + aggregateInteractionEvents(chunks, startTime), + }, ); } } @@ -372,7 +345,7 @@ function bindCurrentSpanStoreToStart< tracingChannel: IsoTracingChannel>, states: WeakMap, create: (event: StartOf) => SpanState, -): (() => void) | undefined { +): void { const state = _internalGetGlobalState(); const contextManager = state?.contextManager; const startChannel = tracingChannel.start as @@ -381,7 +354,6 @@ function bindCurrentSpanStoreToStart< store: CurrentSpanStore, callback: (event: ChannelMessage) => unknown, ) => void; - unbindStore?: (store: CurrentSpanStore) => void; } & object) | undefined; const currentSpanStore = contextManager @@ -393,7 +365,7 @@ function bindCurrentSpanStoreToStart< : undefined; if (!startChannel?.bindStore || !currentSpanStore) { - return undefined; + return; } startChannel.bindStore(currentSpanStore, (event) => { @@ -402,10 +374,6 @@ function bindCurrentSpanStoreToStart< ).span; return contextManager!.wrapSpanForStore(span); }); - - return () => { - startChannel.unbindStore?.(currentSpanStore); - }; } function logErrorAndEndSpan( @@ -1626,3 +1594,13 @@ function tryToDict(obj: unknown): Record | null { return null; } + +let googleGenAIInstrumentationConsumer: + | GoogleGenAIInstrumentationConsumer + | undefined; + +export function registerGoogleGenAIInstrumentation(): void { + googleGenAIInstrumentationConsumer ??= + new GoogleGenAIInstrumentationConsumer(); + googleGenAIInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/groq-channels.ts b/js/src/instrumentation/providers/groq-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/groq-channels.ts rename to js/src/instrumentation/providers/groq-channels.ts diff --git a/js/src/instrumentation/plugins/groq-plugin.test.ts b/js/src/instrumentation/providers/groq-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/groq-plugin.test.ts rename to js/src/instrumentation/providers/groq-instrumentation.test.ts index a140deb80..3b10736d4 100644 --- a/js/src/instrumentation/plugins/groq-plugin.test.ts +++ b/js/src/instrumentation/providers/groq-instrumentation.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { aggregateGroqChatCompletionChunks, parseGroqMetrics, -} from "./groq-plugin"; +} from "./groq-instrumentation"; describe("parseGroqMetrics", () => { it("merges OpenAI-compatible usage metrics with Groq cache metrics", () => { diff --git a/js/src/instrumentation/plugins/groq-plugin.ts b/js/src/instrumentation/providers/groq-instrumentation.ts similarity index 58% rename from js/src/instrumentation/plugins/groq-plugin.ts rename to js/src/instrumentation/providers/groq-instrumentation.ts index 10b6d2c0a..6f2b85d5f 100644 --- a/js/src/instrumentation/plugins/groq-plugin.ts +++ b/js/src/instrumentation/providers/groq-instrumentation.ts @@ -1,8 +1,6 @@ -import { BasePlugin } from "../core"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute } from "../../../util/index"; import { processInputAttachments } from "../../wrappers/attachment-utils"; @@ -10,63 +8,53 @@ import { getCurrentUnixTimestamp } from "../../util"; import { aggregateChatCompletionChunks, parseMetricsFromUsage, -} from "./openai-plugin"; +} from "./openai-instrumentation"; import { groqChannels } from "./groq-channels"; import type { GroqChatCompletion, GroqChatCompletionChunk, } from "../../vendor-sdk-types/groq"; -export class GroqPlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - traceStreamingChannel(groqChannels.chatCompletionsCreate, { - name: "groq.chat.completions.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { messages, ...metadata } = params; - return { - input: processInputAttachments(messages), - metadata: { ...metadata, provider: "groq" }, - }; - }, - extractOutput: (result) => result?.choices, - extractMetrics: (result, startTime) => { - const metrics = parseGroqMetrics(result); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateGroqChatCompletionChunks, - }), - ); +export function registerGroqInstrumentation(): void { + traceStreamingChannel(groqChannels.chatCompletionsCreate, { + name: "groq.chat.completions.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "groq" }, + }; + }, + extractOutput: (result) => result?.choices, + extractMetrics: (result, startTime) => { + const metrics = parseGroqMetrics(result); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateGroqChatCompletionChunks, + }); - this.unsubscribers.push( - traceAsyncChannel(groqChannels.embeddingsCreate, { - name: "groq.embeddings.create", - type: SpanTypeAttribute.LLM, - extractInput: ([params]) => { - const { input, ...metadata } = params; - return { - input, - metadata: { ...metadata, provider: "groq" }, - }; - }, - extractOutput: (result) => { - const embedding = result?.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetrics: (result) => parseGroqMetrics(result), - }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } + traceAsyncChannel(groqChannels.embeddingsCreate, { + name: "groq.embeddings.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input, + metadata: { ...metadata, provider: "groq" }, + }; + }, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetrics: (result) => parseGroqMetrics(result), + }); } export function parseGroqMetrics( diff --git a/js/src/instrumentation/plugins/harness-agent-context.test.ts b/js/src/instrumentation/providers/harness-agent-context.test.ts similarity index 100% rename from js/src/instrumentation/plugins/harness-agent-context.test.ts rename to js/src/instrumentation/providers/harness-agent-context.test.ts diff --git a/js/src/instrumentation/plugins/huggingface-channels.ts b/js/src/instrumentation/providers/huggingface-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/huggingface-channels.ts rename to js/src/instrumentation/providers/huggingface-channels.ts diff --git a/js/src/instrumentation/plugins/huggingface-plugin.test.ts b/js/src/instrumentation/providers/huggingface-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/huggingface-plugin.test.ts rename to js/src/instrumentation/providers/huggingface-instrumentation.test.ts index 8b2b80286..6c55e8bbe 100644 --- a/js/src/instrumentation/plugins/huggingface-plugin.test.ts +++ b/js/src/instrumentation/providers/huggingface-instrumentation.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { aggregateChatCompletionChunks, extractResponseMetadata, -} from "./huggingface-plugin"; +} from "./huggingface-instrumentation"; describe("extractResponseMetadata", () => { it("keeps allowlisted response fields", () => { diff --git a/js/src/instrumentation/plugins/huggingface-plugin.ts b/js/src/instrumentation/providers/huggingface-instrumentation.ts similarity index 87% rename from js/src/instrumentation/plugins/huggingface-plugin.ts rename to js/src/instrumentation/providers/huggingface-instrumentation.ts index 797c3bf82..0407adffa 100644 --- a/js/src/instrumentation/plugins/huggingface-plugin.ts +++ b/js/src/instrumentation/providers/huggingface-instrumentation.ts @@ -1,10 +1,8 @@ import { traceAsyncChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; -import { BasePlugin } from "../core"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { getCurrentUnixTimestamp } from "../../util"; import { parseMetricsFromUsage } from "../../openai-utils"; @@ -41,62 +39,54 @@ const RESPONSE_METADATA_ALLOWLIST = new Set([ "object", ]); -export class HuggingFacePlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - traceAsyncChannel(huggingFaceChannels.chatCompletion, { - name: "huggingface.chat_completion", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - extractOutput: (result) => result?.choices, - extractMetadata: (result) => extractResponseMetadata(result), - extractMetrics: (result) => parseMetricsFromUsage(result?.usage), - }), - traceSyncStreamChannel(huggingFaceChannels.chatCompletionStream, { - name: "huggingface.chat_completion_stream", - type: SpanTypeAttribute.LLM, - extractInput: extractChatInputWithMetadata, - patchResult: ({ result, span, startTime }) => - patchChatCompletionStream({ - result, - span, - startTime, - }), - }), - traceAsyncChannel(huggingFaceChannels.textGeneration, { - name: "huggingface.text_generation", - type: SpanTypeAttribute.LLM, - extractInput: extractTextGenerationInputWithMetadata, - extractOutput: (result) => - isObject(result) ? { generated_text: result.generated_text } : result, - extractMetadata: extractTextGenerationMetadata, - extractMetrics: (result) => - extractTextGenerationMetrics(result?.details ?? null), - }), - traceSyncStreamChannel(huggingFaceChannels.textGenerationStream, { - name: "huggingface.text_generation_stream", - type: SpanTypeAttribute.LLM, - extractInput: extractTextGenerationInputWithMetadata, - patchResult: ({ result, span, startTime }) => - patchTextGenerationStream({ - result, - span, - startTime, - }), +export function registerHuggingFaceInstrumentation(): void { + traceAsyncChannel(huggingFaceChannels.chatCompletion, { + name: "huggingface.chat_completion", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + extractOutput: (result) => result?.choices, + extractMetadata: (result) => extractResponseMetadata(result), + extractMetrics: (result) => parseMetricsFromUsage(result?.usage), + }); + traceSyncStreamChannel(huggingFaceChannels.chatCompletionStream, { + name: "huggingface.chat_completion_stream", + type: SpanTypeAttribute.LLM, + extractInput: extractChatInputWithMetadata, + patchResult: ({ result, span, startTime }) => + patchChatCompletionStream({ + result, + span, + startTime, }), - traceAsyncChannel(huggingFaceChannels.featureExtraction, { - name: "huggingface.feature_extraction", - type: SpanTypeAttribute.LLM, - extractInput: extractFeatureExtractionInputWithMetadata, - extractOutput: summarizeFeatureExtractionOutput, - extractMetrics: () => ({}), + }); + traceAsyncChannel(huggingFaceChannels.textGeneration, { + name: "huggingface.text_generation", + type: SpanTypeAttribute.LLM, + extractInput: extractTextGenerationInputWithMetadata, + extractOutput: (result) => + isObject(result) ? { generated_text: result.generated_text } : result, + extractMetadata: extractTextGenerationMetadata, + extractMetrics: (result) => + extractTextGenerationMetrics(result?.details ?? null), + }); + traceSyncStreamChannel(huggingFaceChannels.textGenerationStream, { + name: "huggingface.text_generation_stream", + type: SpanTypeAttribute.LLM, + extractInput: extractTextGenerationInputWithMetadata, + patchResult: ({ result, span, startTime }) => + patchTextGenerationStream({ + result, + span, + startTime, }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } + }); + traceAsyncChannel(huggingFaceChannels.featureExtraction, { + name: "huggingface.feature_extraction", + type: SpanTypeAttribute.LLM, + extractInput: extractFeatureExtractionInputWithMetadata, + extractOutput: summarizeFeatureExtractionOutput, + extractMetrics: () => ({}), + }); } function addProviderMetadata( diff --git a/js/src/instrumentation/plugins/huggingface-transformers-channels.ts b/js/src/instrumentation/providers/huggingface-transformers-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/huggingface-transformers-channels.ts rename to js/src/instrumentation/providers/huggingface-transformers-channels.ts diff --git a/js/src/instrumentation/plugins/huggingface-transformers-plugin.test.ts b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.test.ts similarity index 97% rename from js/src/instrumentation/plugins/huggingface-transformers-plugin.test.ts rename to js/src/instrumentation/providers/huggingface-transformers-instrumentation.test.ts index 667fc19c7..b32e93579 100644 --- a/js/src/instrumentation/plugins/huggingface-transformers-plugin.test.ts +++ b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.test.ts @@ -3,13 +3,13 @@ import { registerHuggingFaceTransformersPipeline, type HuggingFaceTransformersEventContext, } from "./huggingface-transformers-channels"; -import { _exportsForTestingOnly } from "./huggingface-transformers-plugin"; +import { _exportsForTestingOnly } from "./huggingface-transformers-instrumentation"; import type { HuggingFaceTransformersPipeline } from "../../vendor-sdk-types/huggingface-transformers"; const { extractInput, extractMetadata, extractOutput, isSupportedTask } = _exportsForTestingOnly; -describe("HuggingFaceTransformersPlugin extraction", () => { +describe("registerHuggingFaceTransformersInstrumentation extraction", () => { it("normalizes generation and chat payloads", () => { expect(extractInput("text-generation", ["Hello"])).toEqual([ { role: "user", content: "Hello" }, diff --git a/js/src/instrumentation/plugins/huggingface-transformers-plugin.ts b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.ts similarity index 82% rename from js/src/instrumentation/plugins/huggingface-transformers-plugin.ts rename to js/src/instrumentation/providers/huggingface-transformers-instrumentation.ts index 8d41b535a..09e817685 100644 --- a/js/src/instrumentation/plugins/huggingface-transformers-plugin.ts +++ b/js/src/instrumentation/providers/huggingface-transformers-instrumentation.ts @@ -1,5 +1,4 @@ -import { BasePlugin } from "../core"; -import { traceAsyncChannel, unsubscribeAll } from "../core/channel-tracing"; +import { traceAsyncChannel } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import { SpanTypeAttribute, isObject } from "../../../util"; @@ -19,43 +18,37 @@ const REQUEST_METADATA_KEYS = [ "stop", ] as const; -export class HuggingFaceTransformersPlugin extends BasePlugin { - protected onEnable(): void { +class HuggingFaceTransformersInstrumentationConsumer { + public register(): void { this.subscribeToPipelineFactory(); - this.unsubscribers.push( - traceAsyncChannel(huggingFaceTransformersChannels.pipelineCall, { - name: (_args, event) => { - const task = getTask(event as HuggingFaceTransformersEventContext); - const operation = task?.replaceAll("-", "_") ?? "unknown"; - return `huggingface.transformers.${operation}`; - }, - type: SpanTypeAttribute.LLM, - shouldTrace: (_args, event) => - isSupportedHuggingFaceTransformersTask( - getTask(event as HuggingFaceTransformersEventContext), - ), - extractInput: (args, event) => ({ - input: extractInput( - getTask(event as HuggingFaceTransformersEventContext), - args, - ), - metadata: extractMetadata( - event as HuggingFaceTransformersEventContext, - args, - ), - }), - extractOutput: (result, event) => - extractOutput( - getTask(event as HuggingFaceTransformersEventContext), - result, - ), - extractMetrics: () => ({}), + traceAsyncChannel(huggingFaceTransformersChannels.pipelineCall, { + name: (_args, event) => { + const task = getTask(event as HuggingFaceTransformersEventContext); + const operation = task?.replaceAll("-", "_") ?? "unknown"; + return `huggingface.transformers.${operation}`; + }, + type: SpanTypeAttribute.LLM, + shouldTrace: (_args, event) => + isSupportedHuggingFaceTransformersTask( + getTask(event as HuggingFaceTransformersEventContext), + ), + extractInput: (args, event) => ({ + input: extractInput( + getTask(event as HuggingFaceTransformersEventContext), + args, + ), + metadata: extractMetadata( + event as HuggingFaceTransformersEventContext, + args, + ), }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); + extractOutput: (result, event) => + extractOutput( + getTask(event as HuggingFaceTransformersEventContext), + result, + ), + extractMetrics: () => ({}), + }); } private subscribeToPipelineFactory(): void { @@ -79,7 +72,6 @@ export class HuggingFaceTransformersPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => channel.unsubscribe(handlers)); } } @@ -321,3 +313,13 @@ export const _exportsForTestingOnly = { extractOutput, isSupportedTask: isSupportedHuggingFaceTransformersTask, }; + +let huggingFaceTransformersInstrumentationConsumer: + | HuggingFaceTransformersInstrumentationConsumer + | undefined; + +export function registerHuggingFaceTransformersInstrumentation(): void { + huggingFaceTransformersInstrumentationConsumer ??= + new HuggingFaceTransformersInstrumentationConsumer(); + huggingFaceTransformersInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/instrumentation-names.test.ts b/js/src/instrumentation/providers/instrumentation-names.test.ts similarity index 97% rename from js/src/instrumentation/plugins/instrumentation-names.test.ts rename to js/src/instrumentation/providers/instrumentation-names.test.ts index d8f4418b8..5b0585839 100644 --- a/js/src/instrumentation/plugins/instrumentation-names.test.ts +++ b/js/src/instrumentation/providers/instrumentation-names.test.ts @@ -14,7 +14,6 @@ import { cloudflareAgentsChannels } from "./cloudflare-agents-channels"; import { cloudflareThinkChannels } from "./cloudflare-think-channels"; import { cohereChannels } from "./cohere-channels"; import { cursorSDKChannels } from "./cursor-sdk-channels"; -import { flueChannels } from "./flue-channels"; import { genkitChannels, genkitCoreChannels } from "./genkit-channels"; import { gitHubCopilotChannels } from "./github-copilot-channels"; import { googleADKChannels } from "./google-adk-channels"; @@ -56,7 +55,6 @@ describe("built-in instrumentation provenance names", () => { ], [cohereChannels.chat, INSTRUMENTATION_NAMES.COHERE], [cursorSDKChannels.create, INSTRUMENTATION_NAMES.CURSOR_SDK], - [flueChannels.createContext, INSTRUMENTATION_NAMES.FLUE], [genkitChannels.generate, INSTRUMENTATION_NAMES.GENKIT], [genkitCoreChannels.actionSpan, INSTRUMENTATION_NAMES.GENKIT], [gitHubCopilotChannels.createSession, INSTRUMENTATION_NAMES.GITHUB_COPILOT], diff --git a/js/src/instrumentation/plugins/langchain-channels.ts b/js/src/instrumentation/providers/langchain-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/langchain-channels.ts rename to js/src/instrumentation/providers/langchain-channels.ts diff --git a/js/src/instrumentation/plugins/langchain-plugin.test.ts b/js/src/instrumentation/providers/langchain-instrumentation.test.ts similarity index 79% rename from js/src/instrumentation/plugins/langchain-plugin.test.ts rename to js/src/instrumentation/providers/langchain-instrumentation.test.ts index 29ace20ab..570972a9d 100644 --- a/js/src/instrumentation/plugins/langchain-plugin.test.ts +++ b/js/src/instrumentation/providers/langchain-instrumentation.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest"; -import { LangChainPlugin } from "./langchain-plugin"; +import { beforeAll, describe, expect, it } from "vitest"; +import { registerLangChainInstrumentation } from "./langchain-instrumentation"; import { langChainChannels } from "./langchain-channels"; function createManager(handlers: unknown[] = []) { @@ -35,14 +35,15 @@ function createArgumentsObject(...args: unknown[]): IArguments { })(...args); } -describe("LangChainPlugin", () => { +describe("registerLangChainInstrumentation", () => { + beforeAll(() => { + registerLangChainInstrumentation(); + }); + it("injects a Braintrust callback handler into empty CallbackManager.configure() arguments", () => { - const plugin = new LangChainPlugin(); const args: unknown[] = []; - plugin.enable(); traceConfigureArguments(args); - plugin.disable(); expect(args[0]).toEqual([ expect.objectContaining({ @@ -52,12 +53,9 @@ describe("LangChainPlugin", () => { }); it("injects a Braintrust callback handler into real arguments objects", () => { - const plugin = new LangChainPlugin(); const args = createArgumentsObject(); - plugin.enable(); traceConfigureArgumentsObject(args); - plugin.disable(); expect(args[0]).toEqual([ expect.objectContaining({ @@ -67,12 +65,9 @@ describe("LangChainPlugin", () => { }); it("injects a Braintrust callback handler into CallbackManager.configure() results", () => { - const plugin = new LangChainPlugin(); const manager = createManager(); - plugin.enable(); traceConfigureResult(manager); - plugin.disable(); expect(manager.handlers).toHaveLength(1); expect(manager.handlers[0]).toMatchObject({ @@ -81,37 +76,25 @@ describe("LangChainPlugin", () => { }); it("does not inject duplicate handlers into the same manager", () => { - const plugin = new LangChainPlugin(); const manager = createManager(); - plugin.enable(); traceConfigureResult(manager); traceConfigureResult(manager); - plugin.disable(); expect(manager.handlers).toHaveLength(1); }); it("does not inject when a Braintrust callback handler is already present", () => { - const plugin = new LangChainPlugin(); const existingHandler = { name: "BraintrustCallbackHandler" }; const manager = createManager([existingHandler]); - plugin.enable(); traceConfigureResult(manager); - plugin.disable(); expect(manager.handlers).toEqual([existingHandler]); }); it("gracefully ignores undefined and non-manager results", () => { - const plugin = new LangChainPlugin(); - - plugin.enable(); - expect(() => traceConfigureResult(undefined)).not.toThrow(); expect(() => traceConfigureResult({ handlers: [] })).not.toThrow(); - - plugin.disable(); }); }); diff --git a/js/src/instrumentation/plugins/langchain-plugin.ts b/js/src/instrumentation/providers/langchain-instrumentation.ts similarity index 89% rename from js/src/instrumentation/plugins/langchain-plugin.ts rename to js/src/instrumentation/providers/langchain-instrumentation.ts index 85ef66083..42a5f7fd5 100644 --- a/js/src/instrumentation/plugins/langchain-plugin.ts +++ b/js/src/instrumentation/providers/langchain-instrumentation.ts @@ -1,4 +1,3 @@ -import { BasePlugin } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers, IsoTracingChannel } from "../../isomorph"; import type { LangChainCallbackManager } from "../../vendor-sdk-types/langchain"; @@ -12,22 +11,14 @@ type LangChainConfigureChannel = | typeof langChainChannels.configure | typeof langChainChannels.configureSync; -export class LangChainPlugin extends BasePlugin { +class LangChainInstrumentationConsumer { private injectedManagers = new WeakSet(); - protected onEnable(): void { + public register(): void { this.subscribeToConfigure(langChainChannels.configure); this.subscribeToConfigure(langChainChannels.configureSync); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - this.injectedManagers = new WeakSet(); - } - private subscribeToConfigure(channel: LangChainConfigureChannel): void { const tracingChannel: IsoTracingChannel< ChannelMessage @@ -45,9 +36,6 @@ export class LangChainPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - tracingChannel.unsubscribe(handlers); - }); } private injectHandler(result: unknown): void { @@ -126,3 +114,12 @@ function isBraintrustHandler(handler: unknown): boolean { Reflect.get(handler, "name") === BRAINTRUST_LANGCHAIN_CALLBACK_HANDLER_NAME ); } + +let langChainInstrumentationConsumer: + | LangChainInstrumentationConsumer + | undefined; + +export function registerLangChainInstrumentation(): void { + langChainInstrumentationConsumer ??= new LangChainInstrumentationConsumer(); + langChainInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/langsmith-channels.ts b/js/src/instrumentation/providers/langsmith-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/langsmith-channels.ts rename to js/src/instrumentation/providers/langsmith-channels.ts diff --git a/js/src/instrumentation/plugins/langsmith-plugin.test.ts b/js/src/instrumentation/providers/langsmith-instrumentation.test.ts similarity index 88% rename from js/src/instrumentation/plugins/langsmith-plugin.test.ts rename to js/src/instrumentation/providers/langsmith-instrumentation.test.ts index 8e9b3aed4..55a986a74 100644 --- a/js/src/instrumentation/plugins/langsmith-plugin.test.ts +++ b/js/src/instrumentation/providers/langsmith-instrumentation.test.ts @@ -1,37 +1,36 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { configureInstrumentation } from "../registry"; import { langSmithChannels } from "./langsmith-channels"; -import { LangSmithPlugin } from "./langsmith-plugin"; +import { registerLangSmithInstrumentation } from "./langsmith-instrumentation"; try { + configureInstrumentation({ integrations: { langsmith: false } }); configureNode(); } catch { // Best-effort initialization for test environments. } -describe("LangSmithPlugin", () => { +describe("registerLangSmithInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; - let plugin: LangSmithPlugin; beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); + registerLangSmithInstrumentation(); }); beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "langsmith-plugin.test.ts", + projectName: "langsmith-instrumentation.test.ts", projectId: "test-project-id", }); - plugin = new LangSmithPlugin(); - plugin.enable(); }); afterEach(() => { - plugin.disable(); _exportsForTestingOnly.clearTestBackgroundLogger(); }); @@ -269,7 +268,7 @@ describe("LangSmithPlugin", () => { expect(await backgroundLogger.drain()).toEqual([]); }); - it("suppresses LangChain-serialized runs only when configured", async () => { + it("suppresses LangChain-serialized runs by default", async () => { const run = { id: "44444444-4444-4444-8444-444444444444", trace_id: "44444444-4444-4444-8444-444444444444", @@ -282,31 +281,5 @@ describe("LangSmithPlugin", () => { arguments: [run.id, run], }); expect(await backgroundLogger.drain()).toEqual([]); - - plugin.disable(); - plugin = new LangSmithPlugin({ skipLangChainRuns: false }); - plugin.enable(); - await langSmithChannels.updateRun.tracePromise(async () => undefined, { - arguments: [run.id, run], - }); - expect(await backgroundLogger.drain()).toHaveLength(1); - }); - - it("bounds the completed-run deduplication cache", () => { - const completedRuns = ( - plugin as unknown as { - completedRuns: { - get(key: string): true | undefined; - set(key: string, value: true): void; - }; - } - ).completedRuns; - - for (let index = 0; index <= 10_000; index++) { - completedRuns.set(`run-${index}`, true); - } - - expect(completedRuns.get("run-0")).toBeUndefined(); - expect(completedRuns.get("run-10000")).toBe(true); }); }); diff --git a/js/src/instrumentation/plugins/langsmith-plugin.ts b/js/src/instrumentation/providers/langsmith-instrumentation.ts similarity index 95% rename from js/src/instrumentation/plugins/langsmith-plugin.ts rename to js/src/instrumentation/providers/langsmith-instrumentation.ts index 5027fa694..1716b88cb 100644 --- a/js/src/instrumentation/plugins/langsmith-plugin.ts +++ b/js/src/instrumentation/providers/langsmith-instrumentation.ts @@ -11,8 +11,6 @@ import type { LangSmithBatchIngestRuns, LangSmithRun, } from "../../vendor-sdk-types/langsmith"; -import { BasePlugin } from "../core"; -import { unsubscribeAll } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { langSmithChannels } from "./langsmith-channels"; @@ -21,7 +19,7 @@ type ActiveRun = { span: Span; }; -type LangSmithPluginOptions = { +type LangSmithInstrumentationOptions = { skipLangChainRuns?: boolean; }; @@ -43,19 +41,18 @@ const LLM_SETTING_KEYS = [ "response_format", ] as const; -export class LangSmithPlugin extends BasePlugin { +class LangSmithInstrumentationConsumer { private readonly activeRuns = new Map(); private readonly completedRuns = new LRUCache({ max: MAX_COMPLETED_RUNS, }); private readonly skipLangChainRuns: boolean; - constructor(options: LangSmithPluginOptions = {}) { - super(); + constructor(options: LangSmithInstrumentationOptions = {}) { this.skipLangChainRuns = options.skipLangChainRuns ?? true; } - protected onEnable(): void { + public register(): void { const createChannel = langSmithChannels.createRun.tracingChannel(); const createHandlers = { start: ( @@ -67,7 +64,6 @@ export class LangSmithPlugin extends BasePlugin { }, }; createChannel.subscribe(createHandlers); - this.unsubscribers.push(() => createChannel.unsubscribe(createHandlers)); const updateChannel = langSmithChannels.updateRun.tracingChannel(); const updateHandlers = { @@ -80,7 +76,6 @@ export class LangSmithPlugin extends BasePlugin { }, }; updateChannel.subscribe(updateHandlers); - this.unsubscribers.push(() => updateChannel.unsubscribe(updateHandlers)); const batchChannel = langSmithChannels.batchIngestRuns.tracingChannel(); const batchHandlers = { @@ -93,16 +88,6 @@ export class LangSmithPlugin extends BasePlugin { }, }; batchChannel.subscribe(batchHandlers); - this.unsubscribers.push(() => batchChannel.unsubscribe(batchHandlers)); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - for (const { span } of this.activeRuns.values()) { - span.end(); - } - this.activeRuns.clear(); - this.completedRuns.clear(); } private processBatch(batch: LangSmithBatchIngestRuns): void { @@ -337,6 +322,19 @@ function mergeRuns( return Object.fromEntries(entries); } +let langSmithInstrumentationConsumer: + | LangSmithInstrumentationConsumer + | undefined; + +export function registerLangSmithInstrumentation( + options: LangSmithInstrumentationOptions = {}, +): void { + langSmithInstrumentationConsumer ??= new LangSmithInstrumentationConsumer( + options, + ); + langSmithInstrumentationConsumer.register(); +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/js/src/instrumentation/plugins/mistral-channels.ts b/js/src/instrumentation/providers/mistral-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/mistral-channels.ts rename to js/src/instrumentation/providers/mistral-channels.ts diff --git a/js/src/instrumentation/plugins/mistral-plugin.test.ts b/js/src/instrumentation/providers/mistral-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/mistral-plugin.test.ts rename to js/src/instrumentation/providers/mistral-instrumentation.test.ts index 96d410440..adab7b87b 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.test.ts +++ b/js/src/instrumentation/providers/mistral-instrumentation.test.ts @@ -4,7 +4,7 @@ import { extractMistralRequestMetadata, extractMistralResponseMetadata, parseMistralMetricsFromUsage, -} from "./mistral-plugin"; +} from "./mistral-instrumentation"; describe("extractMistralRequestMetadata", () => { it("keeps only allowlisted request metadata", () => { diff --git a/js/src/instrumentation/plugins/mistral-plugin.ts b/js/src/instrumentation/providers/mistral-instrumentation.ts similarity index 77% rename from js/src/instrumentation/plugins/mistral-plugin.ts rename to js/src/instrumentation/providers/mistral-instrumentation.ts index f33f82554..7179b569d 100644 --- a/js/src/instrumentation/plugins/mistral-plugin.ts +++ b/js/src/instrumentation/providers/mistral-instrumentation.ts @@ -1,8 +1,6 @@ -import { BasePlugin } from "../core"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { SpanTypeAttribute, isObject } from "../../../util/index"; import { processInputAttachments } from "../../wrappers/attachment-utils"; @@ -19,156 +17,130 @@ import type { MistralToolCallDelta, } from "../../vendor-sdk-types/mistral"; -export class MistralPlugin extends BasePlugin { - protected onEnable(): void { +class MistralInstrumentationConsumer { + public register(): void { this.subscribeToMistralChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToMistralChannels(): void { - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.chatComplete, { - name: "mistral.chat.complete", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: (result) => { - return result?.choices; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralMetrics(result?.usage, startTime), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.chatStream, { - name: "mistral.chat.stream", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: extractMistralStreamOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralStreamingMetrics(result, startTime), - aggregateChunks: aggregateMistralStreamChunks, - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.embeddingsCreate, { - name: "mistral.embeddings.create", - type: SpanTypeAttribute.LLM, - extractInput: extractEmbeddingInputWithMetadata, - extractOutput: (result) => { - const embedding = result?.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersModerate, { - name: "mistral.classifiers.moderate", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersModerateChat, { - name: "mistral.classifiers.moderateChat", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersClassify, { - name: "mistral.classifiers.classify", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(mistralChannels.classifiersClassifyChat, { - name: "mistral.classifiers.classifyChat", - type: SpanTypeAttribute.LLM, - extractInput: extractClassifierInputWithMetadata, - extractOutput: extractClassifierOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.fimComplete, { - name: "mistral.fim.complete", - type: SpanTypeAttribute.LLM, - extractInput: extractPromptInputWithMetadata, - extractOutput: (result) => { - return result?.choices; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralMetrics(result?.usage, startTime), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.fimStream, { - name: "mistral.fim.stream", - type: SpanTypeAttribute.LLM, - extractInput: extractPromptInputWithMetadata, - extractOutput: extractMistralStreamOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralStreamingMetrics(result, startTime), - aggregateChunks: aggregateMistralStreamChunks, - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.agentsComplete, { - name: "mistral.agents.complete", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: (result) => { - return result?.choices; - }, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralMetrics(result?.usage, startTime), - }), - ); - - this.unsubscribers.push( - traceStreamingChannel(mistralChannels.agentsStream, { - name: "mistral.agents.stream", - type: SpanTypeAttribute.LLM, - extractInput: extractMessagesInputWithMetadata, - extractOutput: extractMistralStreamOutput, - extractMetadata: (result) => extractMistralResponseMetadata(result), - extractMetrics: (result, startTime) => - extractMistralStreamingMetrics(result, startTime), - aggregateChunks: aggregateMistralStreamChunks, - }), - ); + traceStreamingChannel(mistralChannels.chatComplete, { + name: "mistral.chat.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }); + + traceStreamingChannel(mistralChannels.chatStream, { + name: "mistral.chat.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }); + + traceAsyncChannel(mistralChannels.embeddingsCreate, { + name: "mistral.embeddings.create", + type: SpanTypeAttribute.LLM, + extractInput: extractEmbeddingInputWithMetadata, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersModerate, { + name: "mistral.classifiers.moderate", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersModerateChat, { + name: "mistral.classifiers.moderateChat", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersClassify, { + name: "mistral.classifiers.classify", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceAsyncChannel(mistralChannels.classifiersClassifyChat, { + name: "mistral.classifiers.classifyChat", + type: SpanTypeAttribute.LLM, + extractInput: extractClassifierInputWithMetadata, + extractOutput: extractClassifierOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result) => parseMistralMetricsFromUsage(result?.usage), + }); + + traceStreamingChannel(mistralChannels.fimComplete, { + name: "mistral.fim.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }); + + traceStreamingChannel(mistralChannels.fimStream, { + name: "mistral.fim.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractPromptInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }); + + traceStreamingChannel(mistralChannels.agentsComplete, { + name: "mistral.agents.complete", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: (result) => { + return result?.choices; + }, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralMetrics(result?.usage, startTime), + }); + + traceStreamingChannel(mistralChannels.agentsStream, { + name: "mistral.agents.stream", + type: SpanTypeAttribute.LLM, + extractInput: extractMessagesInputWithMetadata, + extractOutput: extractMistralStreamOutput, + extractMetadata: (result) => extractMistralResponseMetadata(result), + extractMetrics: (result, startTime) => + extractMistralStreamingMetrics(result, startTime), + aggregateChunks: aggregateMistralStreamChunks, + }); } } @@ -909,3 +881,10 @@ export function aggregateMistralStreamChunks( ...(metadata ? { metadata } : {}), }; } + +let mistralInstrumentationConsumer: MistralInstrumentationConsumer | undefined; + +export function registerMistralInstrumentation(): void { + mistralInstrumentationConsumer ??= new MistralInstrumentationConsumer(); + mistralInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/ollama-channels.ts b/js/src/instrumentation/providers/ollama-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/ollama-channels.ts rename to js/src/instrumentation/providers/ollama-channels.ts diff --git a/js/src/instrumentation/plugins/ollama-plugin.test.ts b/js/src/instrumentation/providers/ollama-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/ollama-plugin.test.ts rename to js/src/instrumentation/providers/ollama-instrumentation.test.ts index 0006efea8..4503d4ddf 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.test.ts +++ b/js/src/instrumentation/providers/ollama-instrumentation.test.ts @@ -9,7 +9,7 @@ import { extractOllamaEmbedOutput, extractOllamaGenerateInput, extractOllamaMetrics, -} from "./ollama-plugin"; +} from "./ollama-instrumentation"; configureNode(); diff --git a/js/src/instrumentation/plugins/ollama-plugin.ts b/js/src/instrumentation/providers/ollama-instrumentation.ts similarity index 91% rename from js/src/instrumentation/plugins/ollama-plugin.ts rename to js/src/instrumentation/providers/ollama-instrumentation.ts index f48040348..405b3dff9 100644 --- a/js/src/instrumentation/plugins/ollama-plugin.ts +++ b/js/src/instrumentation/providers/ollama-instrumentation.ts @@ -14,54 +14,44 @@ import type { OllamaToolCall, OllamaUsageResponse, } from "../../vendor-sdk-types/ollama"; -import { BasePlugin } from "../core"; import type { AsyncEndOf } from "../core/channel-definitions"; import { traceAsyncChannel, traceStreamingChannel, - unsubscribeAll, } from "../core/channel-tracing"; import { ollamaChannels } from "./ollama-channels"; -export class OllamaPlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - traceStreamingChannel(ollamaChannels.chat, { - name: "ollama.chat", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaChatInput, - extractOutput: (result, event) => - extractOllamaChatOutput( - result, - countOllamaToolCalls(event?.arguments?.[0]?.messages), - ), - extractMetadata: extractOllamaResponseMetadata, - extractMetrics: extractOllamaMetrics, - aggregateChunks: aggregateOllamaChatChunks, - }), - traceStreamingChannel(ollamaChannels.generate, { - name: "ollama.generate", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaGenerateInput, - extractOutput: extractOllamaGenerateOutput, - extractMetadata: extractOllamaResponseMetadata, - extractMetrics: extractOllamaMetrics, - aggregateChunks: aggregateOllamaGenerateChunks, - }), - traceAsyncChannel(ollamaChannels.embed, { - name: "ollama.embed", - type: SpanTypeAttribute.LLM, - extractInput: extractOllamaEmbedInput, - extractOutput: extractOllamaEmbedOutput, - extractMetadata: extractOllamaResponseMetadata, - extractMetrics: extractOllamaMetrics, - }), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } +export function registerOllamaInstrumentation(): void { + traceStreamingChannel(ollamaChannels.chat, { + name: "ollama.chat", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaChatInput, + extractOutput: (result, event) => + extractOllamaChatOutput( + result, + countOllamaToolCalls(event?.arguments?.[0]?.messages), + ), + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaChatChunks, + }); + traceStreamingChannel(ollamaChannels.generate, { + name: "ollama.generate", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaGenerateInput, + extractOutput: extractOllamaGenerateOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + aggregateChunks: aggregateOllamaGenerateChunks, + }); + traceAsyncChannel(ollamaChannels.embed, { + name: "ollama.embed", + type: SpanTypeAttribute.LLM, + extractInput: extractOllamaEmbedInput, + extractOutput: extractOllamaEmbedOutput, + extractMetadata: extractOllamaResponseMetadata, + extractMetrics: extractOllamaMetrics, + }); } function isNonNegativeNumber(value: unknown): value is number { diff --git a/js/src/instrumentation/plugins/openai-agents-channels.ts b/js/src/instrumentation/providers/openai-agents-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-agents-channels.ts rename to js/src/instrumentation/providers/openai-agents-channels.ts diff --git a/js/src/instrumentation/plugins/openai-agents-plugin.test.ts b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts similarity index 88% rename from js/src/instrumentation/plugins/openai-agents-plugin.test.ts rename to js/src/instrumentation/providers/openai-agents-instrumentation.test.ts index 9f2a604bd..fb6f62fba 100644 --- a/js/src/instrumentation/plugins/openai-agents-plugin.test.ts +++ b/js/src/instrumentation/providers/openai-agents-instrumentation.test.ts @@ -1,37 +1,36 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, initLogger } from "../../logger"; +import { configureInstrumentation } from "../registry"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; -import { OpenAIAgentsPlugin } from "./openai-agents-plugin"; +import { registerOpenAIAgentsInstrumentation } from "./openai-agents-instrumentation"; try { + configureInstrumentation({ integrations: { openAIAgents: false } }); configureNode(); } catch { // Best-effort initialization for test environments. } -describe("OpenAIAgentsPlugin", () => { +describe("registerOpenAIAgentsInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; - let plugin: OpenAIAgentsPlugin; beforeAll(async () => { await _exportsForTestingOnly.simulateLoginForTests(); + registerOpenAIAgentsInstrumentation(); }); beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "openai-agents-plugin.test.ts", + projectName: "openai-agents-instrumentation.test.ts", projectId: "test-project-id", }); - plugin = new OpenAIAgentsPlugin(); - plugin.enable(); }); afterEach(() => { - plugin.disable(); _exportsForTestingOnly.clearTestBackgroundLogger(); }); diff --git a/js/src/instrumentation/plugins/openai-agents-plugin.ts b/js/src/instrumentation/providers/openai-agents-instrumentation.ts similarity index 80% rename from js/src/instrumentation/plugins/openai-agents-plugin.ts rename to js/src/instrumentation/providers/openai-agents-instrumentation.ts index 4cf8af1a0..9db90ac4c 100644 --- a/js/src/instrumentation/plugins/openai-agents-plugin.ts +++ b/js/src/instrumentation/providers/openai-agents-instrumentation.ts @@ -1,5 +1,3 @@ -import { BasePlugin } from "../core"; -import { unsubscribeAll } from "../core/channel-tracing"; import { isObject } from "../../../util/index"; import { openAIAgentsCoreChannels } from "./openai-agents-channels"; import { OpenAIAgentsTraceProcessor } from "./openai-agents-trace-processor"; @@ -41,18 +39,13 @@ function isOpenAIAgentsSpan(value: unknown): value is OpenAIAgentsSpan { ); } -export class OpenAIAgentsPlugin extends BasePlugin { +class OpenAIAgentsInstrumentationConsumer { private processor = new OpenAIAgentsTraceProcessor(); - protected onEnable(): void { + public register(): void { this.subscribeToTraceLifecycle(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - void this.processor.shutdown(); - } - private subscribeToTraceLifecycle(): void { const traceStartChannel = openAIAgentsCoreChannels.onTraceStart.tracingChannel(); @@ -65,9 +58,6 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; traceStartChannel.subscribe(traceStartHandlers); - this.unsubscribers.push(() => - traceStartChannel.unsubscribe(traceStartHandlers), - ); const traceEndChannel = openAIAgentsCoreChannels.onTraceEnd.tracingChannel(); @@ -80,9 +70,6 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; traceEndChannel.subscribe(traceEndHandlers); - this.unsubscribers.push(() => - traceEndChannel.unsubscribe(traceEndHandlers), - ); const spanStartChannel = openAIAgentsCoreChannels.onSpanStart.tracingChannel(); @@ -95,9 +82,6 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; spanStartChannel.subscribe(spanStartHandlers); - this.unsubscribers.push(() => - spanStartChannel.unsubscribe(spanStartHandlers), - ); const spanEndChannel = openAIAgentsCoreChannels.onSpanEnd.tracingChannel(); const spanEndHandlers = { @@ -109,6 +93,15 @@ export class OpenAIAgentsPlugin extends BasePlugin { }, }; spanEndChannel.subscribe(spanEndHandlers); - this.unsubscribers.push(() => spanEndChannel.unsubscribe(spanEndHandlers)); } } + +let openAIAgentsInstrumentationConsumer: + | OpenAIAgentsInstrumentationConsumer + | undefined; + +export function registerOpenAIAgentsInstrumentation(): void { + openAIAgentsInstrumentationConsumer ??= + new OpenAIAgentsInstrumentationConsumer(); + openAIAgentsInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/openai-agents-trace-processor.ts b/js/src/instrumentation/providers/openai-agents-trace-processor.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-agents-trace-processor.ts rename to js/src/instrumentation/providers/openai-agents-trace-processor.ts diff --git a/js/src/instrumentation/plugins/openai-channels.ts b/js/src/instrumentation/providers/openai-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-channels.ts rename to js/src/instrumentation/providers/openai-channels.ts diff --git a/js/src/instrumentation/plugins/openai-codex-channels.ts b/js/src/instrumentation/providers/openai-codex-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openai-codex-channels.ts rename to js/src/instrumentation/providers/openai-codex-channels.ts diff --git a/js/src/instrumentation/plugins/openai-codex-plugin.test.ts b/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts similarity index 92% rename from js/src/instrumentation/plugins/openai-codex-plugin.test.ts rename to js/src/instrumentation/providers/openai-codex-instrumentation.test.ts index 932abe0ba..f7250bb3b 100644 --- a/js/src/instrumentation/plugins/openai-codex-plugin.test.ts +++ b/js/src/instrumentation/providers/openai-codex-instrumentation.test.ts @@ -15,11 +15,11 @@ vi.mock("../../logger", () => ({ })); import iso from "../../isomorph"; -import { OpenAICodexPlugin } from "./openai-codex-plugin"; +import { registerOpenAICodexInstrumentation } from "./openai-codex-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("OpenAICodexPlugin", () => { +describe("registerOpenAICodexInstrumentation", () => { let handlersByName: Map; let spans: Array<{ end: ReturnType; @@ -33,7 +33,6 @@ describe("OpenAICodexPlugin", () => { spans = []; mockNewTracingChannel.mockImplementation((name: string) => ({ subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), - unsubscribe: vi.fn(), })); mockStartSpan.mockImplementation((args: any) => { const span = { @@ -55,8 +54,7 @@ describe("OpenAICodexPlugin", () => { }); it("uses explicit token totals by precedence instead of summing breakdowns", async () => { - const plugin = new OpenAICodexPlugin(); - plugin.enable(); + registerOpenAICodexInstrumentation(); const runHandlers = handlersByName.get( "orchestrion:@openai/codex-sdk:Thread.run", diff --git a/js/src/instrumentation/plugins/openai-codex-plugin.ts b/js/src/instrumentation/providers/openai-codex-instrumentation.ts similarity index 98% rename from js/src/instrumentation/plugins/openai-codex-plugin.ts rename to js/src/instrumentation/providers/openai-codex-instrumentation.ts index 4005ecb86..abf442120 100644 --- a/js/src/instrumentation/plugins/openai-codex-plugin.ts +++ b/js/src/instrumentation/providers/openai-codex-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import type { IsoChannelHandlers } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; @@ -55,19 +55,12 @@ const PATCHED_STREAMED_TURN = Symbol.for( "braintrust.openai-codex.patched-streamed-turn", ); -export class OpenAICodexPlugin extends BasePlugin { - protected onEnable(): void { +class OpenAICodexInstrumentationConsumer { + public register(): void { this.subscribeToRun(); this.subscribeToRunStreamed(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToRun(): void { const channel = openAICodexChannels.run.tracingChannel(); const states = new WeakMap(); @@ -97,9 +90,6 @@ export class OpenAICodexPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } private subscribeToRunStreamed(): void { @@ -131,9 +121,6 @@ export class OpenAICodexPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - channel.unsubscribe(handlers); - }); } } @@ -941,3 +928,13 @@ function safeLog(span: Span, event: Parameters[0]): void { function logInstrumentationError(context: string, error: unknown): void { debugLogger.error(`Error processing ${context}:`, error); } + +let openAICodexInstrumentationConsumer: + | OpenAICodexInstrumentationConsumer + | undefined; + +export function registerOpenAICodexInstrumentation(): void { + openAICodexInstrumentationConsumer ??= + new OpenAICodexInstrumentationConsumer(); + openAICodexInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/openai-plugin.test.ts b/js/src/instrumentation/providers/openai-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/openai-plugin.test.ts rename to js/src/instrumentation/providers/openai-instrumentation.test.ts index 5f2b237fa..d51301e85 100644 --- a/js/src/instrumentation/plugins/openai-plugin.test.ts +++ b/js/src/instrumentation/providers/openai-instrumentation.test.ts @@ -3,7 +3,7 @@ import { parseMetricsFromUsage, processImagesInOutput, aggregateChatCompletionChunks, -} from "./openai-plugin"; +} from "./openai-instrumentation"; import { Attachment } from "../../logger"; describe("parseMetricsFromUsage", () => { diff --git a/js/src/instrumentation/providers/openai-instrumentation.ts b/js/src/instrumentation/providers/openai-instrumentation.ts new file mode 100644 index 000000000..30cddb5c2 --- /dev/null +++ b/js/src/instrumentation/providers/openai-instrumentation.ts @@ -0,0 +1,573 @@ +import { + traceAsyncChannel, + traceStreamingChannel, + traceSyncStreamChannel, +} from "../core/channel-tracing"; +import { Attachment } from "../../logger"; +import { SpanTypeAttribute, isObject } from "../../../util/index"; +import { getCurrentUnixTimestamp } from "../../util"; +import { processInputAttachments } from "../../wrappers/attachment-utils"; +import { openAIChannels } from "./openai-channels"; +import { + BRAINTRUST_CACHED_STREAM_METRIC, + getCachedMetricFromHeaders, + parseMetricsFromUsage, +} from "../../openai-utils"; +import type { + OpenAIChatChoice, + OpenAIChatCompletionChunk, + OpenAIChatLogprobs, + OpenAIResponseStreamEvent, +} from "../../vendor-sdk-types/openai"; + +export function registerOpenAIInstrumentation(): void { + // Chat Completions - supports streaming + traceStreamingChannel(openAIChannels.chatCompletionsCreate, { + name: "Chat Completion", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return result?.choices; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateChatCompletionChunks, + }); + + // Embeddings + traceAsyncChannel(openAIChannels.embeddingsCreate, { + name: "Embedding", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input, + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + const embedding = result?.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetrics: (result, _startTime, endEvent) => { + return withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + }, + }); + + // Beta Chat Completions Parse + traceStreamingChannel(openAIChannels.betaChatCompletionsParse, { + name: "Chat Completion", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return result?.choices; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateChatCompletionChunks, + }); + + // Beta Chat Completions Stream (sync method returning event-based stream) + traceSyncStreamChannel(openAIChannels.betaChatCompletionsStream, { + name: "Chat Completion", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { messages, ...metadata } = params; + return { + input: processInputAttachments(messages), + metadata: { ...metadata, provider: "openai" }, + }; + }, + }); + + // Moderations + traceAsyncChannel(openAIChannels.moderationsCreate, { + name: "Moderation", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input, + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return result?.results; + }, + extractMetrics: (result, _startTime, endEvent) => { + return withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + }, + }); + + // Responses API - create (supports streaming via stream=true param) + traceStreamingChannel(openAIChannels.responsesCreate, { + name: "openai.responses.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateResponseStreamEvents, + }); + + // Responses API - stream (sync method returning event-based stream) + traceSyncStreamChannel(openAIChannels.responsesStream, { + name: "openai.responses.create", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractFromEvent: (event) => { + if (event.type !== "response.completed" || !event.response) { + return {}; + } + + const response = event.response; + const data: Record = {}; + + if (response.output !== undefined) { + data.output = processImagesInOutput(response.output); + } + + const { usage: _usage, output: _output, ...metadata } = response; + if (Object.keys(metadata).length > 0) { + data.metadata = metadata; + } + + data.metrics = parseMetricsFromUsage(response.usage); + return data; + }, + }); + + // Responses API - parse + traceStreamingChannel(openAIChannels.responsesParse, { + name: "openai.responses.parse", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateResponseStreamEvents, + }); + + // Responses API - compact + traceAsyncChannel(openAIChannels.responsesCompact, { + name: "openai.responses.compact", + type: SpanTypeAttribute.LLM, + extractInput: ([params]) => { + const { input, ...metadata } = params; + return { + input: processInputAttachments(input), + metadata: { ...metadata, provider: "openai" }, + }; + }, + extractOutput: (result) => { + return processImagesInOutput(result?.output); + }, + extractMetadata: (result) => { + if (!result) { + return undefined; + } + const { output: _output, usage: _usage, ...metadata } = result; + return Object.keys(metadata).length > 0 ? metadata : undefined; + }, + extractMetrics: (result, startTime, endEvent) => { + const metrics = withCachedMetric( + parseMetricsFromUsage(result?.usage), + result, + endEvent, + ); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + }); +} + +function getCachedMetricFromEndEvent(endEvent: unknown): number | undefined { + if (!isObject(endEvent)) { + return undefined; + } + + const response = (endEvent as Record).response; + if (!isObject(response)) { + return undefined; + } + + const headers = (response as { headers?: unknown }).headers; + if (!headers || typeof (headers as Headers).get !== "function") { + return undefined; + } + + return getCachedMetricFromHeaders(headers as Headers); +} + +function withCachedMetric( + metrics: Record, + result: unknown, + endEvent?: unknown, +): Record { + if (metrics.cached !== undefined) { + return metrics; + } + + const cachedFromEvent = getCachedMetricFromEndEvent(endEvent); + if (cachedFromEvent !== undefined) { + return { + ...metrics, + cached: cachedFromEvent, + }; + } + + if (!isObject(result)) { + return metrics; + } + + const cached = (result as Record)[ + BRAINTRUST_CACHED_STREAM_METRIC + ]; + + if (typeof cached !== "number") { + return metrics; + } + + return { + ...metrics, + cached, + }; +} + +/** + * Process output to convert base64 images to attachments. + * Used for Responses API image generation output. + */ +export function processImagesInOutput(output: any): any { + if (Array.isArray(output)) { + return output.map(processImagesInOutput); + } + + if (isObject(output)) { + if ( + output.type === "image_generation_call" && + output.result && + typeof output.result === "string" + ) { + const fileExtension = output.output_format || "png"; + const contentType = `image/${fileExtension}`; + + const baseFilename = + output.revised_prompt && typeof output.revised_prompt === "string" + ? output.revised_prompt.slice(0, 50).replace(/[^a-zA-Z0-9]/g, "_") + : "generated_image"; + const filename = `${baseFilename}.${fileExtension}`; + + // Convert base64 string to Blob + const binaryString = atob(output.result); + const bytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + const blob = new Blob([bytes], { type: contentType }); + + const attachment = new Attachment({ + data: blob, + filename: filename, + contentType: contentType, + }); + + return { + ...output, + result: attachment, + }; + } + } + + return output; +} + +function mergeLogprobTokens( + existing: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], + incoming: OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"], +): OpenAIChatLogprobs["content"] | OpenAIChatLogprobs["refusal"] { + if (incoming === undefined) { + return existing; + } + + if (incoming === null) { + return existing ?? null; + } + + if (Array.isArray(existing)) { + return [...existing, ...incoming]; + } + + return [...incoming]; +} + +function aggregateChatLogprobs( + existing: OpenAIChatLogprobs | null | undefined, + incoming: OpenAIChatLogprobs | null | undefined, +): OpenAIChatLogprobs | null | undefined { + if (incoming === undefined) { + return existing; + } + + if (incoming === null) { + return existing ?? null; + } + + const aggregated: OpenAIChatLogprobs = + existing && existing !== null + ? { ...existing, ...incoming } + : { ...incoming }; + + const content = mergeLogprobTokens(existing?.content, incoming.content); + if (content !== undefined) { + aggregated.content = content; + } + + const refusal = mergeLogprobTokens(existing?.refusal, incoming.refusal); + if (refusal !== undefined) { + aggregated.refusal = refusal; + } + + return aggregated; +} + +/** + * Aggregate chat completion chunks into a single response. + * Combines role (first), content (concatenated), tool_calls (by id), + * finish_reason (last), and usage (last chunk). + */ +export function aggregateChatCompletionChunks( + chunks: OpenAIChatCompletionChunk[], + streamResult?: unknown, + endEvent?: unknown, +): { + output: OpenAIChatChoice[]; + metrics: Record; +} { + let role = undefined; + let content = undefined; + let refusal = undefined; + let tool_calls = undefined; + let logprobs: OpenAIChatLogprobs | null | undefined = undefined; + let finish_reason = undefined; + let metrics: Record = {}; + + for (const chunk of chunks) { + if (chunk.usage) { + metrics = { + ...metrics, + ...parseMetricsFromUsage(chunk.usage), + }; + } + + const choice = chunk.choices?.[0]; + if (!choice) { + continue; + } + + if (choice.finish_reason) { + finish_reason = choice.finish_reason; + } + + logprobs = aggregateChatLogprobs(logprobs, choice.logprobs); + + const delta = choice.delta; + if (!delta) { + continue; + } + + if (delta.finish_reason) { + finish_reason = delta.finish_reason; + } + + if (!role && delta.role) { + role = delta.role; + } + + if (delta.content) { + content = (content || "") + delta.content; + } + + if (delta.refusal) { + refusal = (refusal || "") + delta.refusal; + } + + if (delta.tool_calls) { + const toolDelta = delta.tool_calls[0]; + if ( + !tool_calls || + (toolDelta.id && tool_calls[tool_calls.length - 1].id !== toolDelta.id) + ) { + tool_calls = [ + ...(tool_calls || []), + { + id: toolDelta.id, + type: toolDelta.type, + function: toolDelta.function, + }, + ]; + } else { + tool_calls[tool_calls.length - 1].function.arguments += + toolDelta.function.arguments; + } + } + } + + metrics = withCachedMetric(metrics, streamResult, endEvent); + + return { + metrics, + output: [ + { + index: 0, + message: { + role, + content, + ...(refusal !== undefined ? { refusal } : {}), + tool_calls, + }, + logprobs: logprobs ?? null, + finish_reason, + }, + ], + }; +} + +function aggregateResponseStreamEvents( + chunks: OpenAIResponseStreamEvent[], + _streamResult?: unknown, + endEvent?: unknown, +): { + output: any; + metrics: Record; + metadata?: Record; +} { + let output: any = undefined; + let metrics: Record = {}; + let metadata: Record | undefined = undefined; + + for (const chunk of chunks) { + if (!chunk || !chunk.type || !chunk.response) { + continue; + } + if (chunk.type !== "response.completed") { + continue; + } + + const response = chunk.response; + if (response?.output !== undefined) { + output = processImagesInOutput(response.output); + } + + const { usage: _usage, output: _output, ...rest } = response || {}; + if (Object.keys(rest).length > 0) { + metadata = rest; + } + + metrics = parseMetricsFromUsage(response?.usage); + } + + return { + output, + metrics: withCachedMetric(metrics, undefined, endEvent), + ...(metadata !== undefined ? { metadata } : {}), + }; +} + +export { parseMetricsFromUsage }; diff --git a/js/src/instrumentation/plugins/openrouter-agent-channels.ts b/js/src/instrumentation/providers/openrouter-agent-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openrouter-agent-channels.ts rename to js/src/instrumentation/providers/openrouter-agent-channels.ts diff --git a/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts b/js/src/instrumentation/providers/openrouter-agent-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts rename to js/src/instrumentation/providers/openrouter-agent-instrumentation.test.ts index 85d66d51c..27fa78ca2 100644 --- a/js/src/instrumentation/plugins/openrouter-agent-plugin.test.ts +++ b/js/src/instrumentation/providers/openrouter-agent-instrumentation.test.ts @@ -14,7 +14,7 @@ import { aggregateOpenRouterChatChunks, aggregateOpenRouterResponseStreamEvents, parseOpenRouterMetricsFromUsage, -} from "./openrouter-agent-plugin"; +} from "./openrouter-agent-instrumentation"; const TEST_PROVIDER = "openai"; const TEST_MODEL = "gpt-4.1-mini"; @@ -25,7 +25,7 @@ try { // Best-effort initialization for test environments. } -describe("OpenRouter Agent Plugin", () => { +describe("OpenRouter Agent instrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -37,7 +37,7 @@ describe("OpenRouter Agent Plugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "openrouter-agent-plugin.test.ts", + projectName: "openrouter-agent-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/openrouter-agent-plugin.ts b/js/src/instrumentation/providers/openrouter-agent-instrumentation.ts similarity index 91% rename from js/src/instrumentation/plugins/openrouter-agent-plugin.ts rename to js/src/instrumentation/providers/openrouter-agent-instrumentation.ts index facda63c0..7f77cccfc 100644 --- a/js/src/instrumentation/plugins/openrouter-agent-plugin.ts +++ b/js/src/instrumentation/providers/openrouter-agent-instrumentation.ts @@ -1,9 +1,8 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { traceAsyncChannel, traceStreamingChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { @@ -26,107 +25,93 @@ import type { OpenRouterAgentToolTurnContext, } from "../../vendor-sdk-types/openrouter-agent"; -export class OpenRouterAgentPlugin extends BasePlugin { - protected onEnable(): void { +class OpenRouterAgentInstrumentationConsumer { + public register(): void { this.subscribeToOpenRouterAgentChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToOpenRouterAgentChannels(): void { - this.unsubscribers.push( - traceSyncStreamChannel(openRouterAgentChannels.callModel, { - name: "openrouter.callModel", - type: SpanTypeAttribute.TASK, - extractInput: (args) => { - const request = getOpenRouterCallModelRequestArg(args); - return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: request - ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }, - }; - }, - patchResult: ({ endEvent, result, span }) => { - return patchOpenRouterCallModelResult({ - request: getOpenRouterCallModelRequestArg(endEvent.arguments), - result, - span, - }); - }, - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(openRouterAgentChannels.callModelTurn, { - name: "openrouter.beta.responses.send", - type: SpanTypeAttribute.LLM, - extractInput: (args, event) => { - const request = getOpenRouterCallModelRequestArg(args); - const metadata = request + traceSyncStreamChannel(openRouterAgentChannels.callModel, { + name: "openrouter.callModel", + type: SpanTypeAttribute.TASK, + extractInput: (args) => { + const request = getOpenRouterCallModelRequestArg(args); + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: request ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }; + : { provider: "openrouter" }, + }; + }, + patchResult: ({ endEvent, result, span }) => { + return patchOpenRouterCallModelResult({ + request: getOpenRouterCallModelRequestArg(endEvent.arguments), + result, + span, + }); + }, + }); - if (isObject(metadata) && "tools" in metadata) { - delete (metadata as Record).tools; - } + traceAsyncChannel(openRouterAgentChannels.callModelTurn, { + name: "openrouter.beta.responses.send", + type: SpanTypeAttribute.LLM, + extractInput: (args, event) => { + const request = getOpenRouterCallModelRequestArg(args); + const metadata = request + ? extractOpenRouterCallModelMetadata(request) + : { provider: "openrouter" }; + + if (isObject(metadata) && "tools" in metadata) { + delete (metadata as Record).tools; + } + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: { + ...metadata, + step: event.step, + step_type: event.stepType, + }, + }; + }, + extractOutput: (result) => + extractOpenRouterResponseOutput(result as Record), + extractMetadata: (result, event) => { + if (!isObject(result)) { return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: { - ...metadata, - step: event.step, - step_type: event.stepType, - }, + step: event?.step, + step_type: event?.stepType, }; - }, - extractOutput: (result) => - extractOpenRouterResponseOutput(result as Record), - extractMetadata: (result, event) => { - if (!isObject(result)) { - return { - step: event?.step, - step_type: event?.stepType, - }; - } + } - return { - ...(extractOpenRouterResponseMetadata(result) || {}), - ...(event?.step !== undefined ? { step: event.step } : {}), - ...(event?.stepType ? { step_type: event.stepType } : {}), - }; + return { + ...(extractOpenRouterResponseMetadata(result) || {}), + ...(event?.step !== undefined ? { step: event.step } : {}), + ...(event?.stepType ? { step_type: event.stepType } : {}), + }; + }, + extractMetrics: (result) => + isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, + }); + + traceStreamingChannel(openRouterAgentChannels.toolExecute, { + name: "openrouter.tool", + type: SpanTypeAttribute.TOOL, + extractInput: (args, event) => ({ + input: args[0], + metadata: { + provider: "openrouter", + tool_name: event.toolName, + ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), }, - extractMetrics: (result) => - isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, }), - ); - - this.unsubscribers.push( - traceStreamingChannel(openRouterAgentChannels.toolExecute, { - name: "openrouter.tool", - type: SpanTypeAttribute.TOOL, - extractInput: (args, event) => ({ - input: args[0], - metadata: { - provider: "openrouter", - tool_name: event.toolName, - ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), - }, - }), - extractOutput: (result) => result, - extractMetrics: () => ({}), - aggregateChunks: (chunks) => ({ - output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, - metrics: {}, - }), + extractOutput: (result) => result, + extractMetrics: () => ({}), + aggregateChunks: (chunks) => ({ + output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, + metrics: {}, }), - ); + }); const callModelChannel = openRouterAgentChannels.callModel.tracingChannel(); const callModelHandlers = { @@ -141,9 +126,6 @@ export class OpenRouterAgentPlugin extends BasePlugin { }; callModelChannel.subscribe(callModelHandlers); - this.unsubscribers.push(() => { - callModelChannel.unsubscribe(callModelHandlers); - }); } } @@ -1326,3 +1308,13 @@ function normalizeError(error: unknown): Error { } export { parseOpenRouterMetricsFromUsage }; + +let openRouterAgentInstrumentationConsumer: + | OpenRouterAgentInstrumentationConsumer + | undefined; + +export function registerOpenRouterAgentInstrumentation(): void { + openRouterAgentInstrumentationConsumer ??= + new OpenRouterAgentInstrumentationConsumer(); + openRouterAgentInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/openrouter-channels.ts b/js/src/instrumentation/providers/openrouter-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/openrouter-channels.ts rename to js/src/instrumentation/providers/openrouter-channels.ts diff --git a/js/src/instrumentation/plugins/openrouter-plugin.test.ts b/js/src/instrumentation/providers/openrouter-instrumentation.test.ts similarity index 99% rename from js/src/instrumentation/plugins/openrouter-plugin.test.ts rename to js/src/instrumentation/providers/openrouter-instrumentation.test.ts index 93627512e..c656d5e6d 100644 --- a/js/src/instrumentation/plugins/openrouter-plugin.test.ts +++ b/js/src/instrumentation/providers/openrouter-instrumentation.test.ts @@ -14,7 +14,7 @@ import { aggregateOpenRouterChatChunks, aggregateOpenRouterResponseStreamEvents, parseOpenRouterMetricsFromUsage, -} from "./openrouter-plugin"; +} from "./openrouter-instrumentation"; const TEST_PROVIDER = "openai"; const TEST_MODEL = "gpt-4.1-mini"; @@ -27,7 +27,7 @@ try { // Best-effort initialization for test environments. } -describe("OpenRouter Plugin", () => { +describe("OpenRouter instrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -39,7 +39,7 @@ describe("OpenRouter Plugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "openrouter-plugin.test.ts", + projectName: "openrouter-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/openrouter-plugin.ts b/js/src/instrumentation/providers/openrouter-instrumentation.ts similarity index 80% rename from js/src/instrumentation/plugins/openrouter-plugin.ts rename to js/src/instrumentation/providers/openrouter-instrumentation.ts index c89b8f9a7..fa8e54aaa 100644 --- a/js/src/instrumentation/plugins/openrouter-plugin.ts +++ b/js/src/instrumentation/providers/openrouter-instrumentation.ts @@ -1,9 +1,8 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import { traceAsyncChannel, traceStreamingChannel, traceSyncStreamChannel, - unsubscribeAll, } from "../core/channel-tracing"; import type { ChannelMessage } from "../core/channel-definitions"; import { @@ -28,247 +27,225 @@ import type { OpenRouterToolTurnContext, } from "../../vendor-sdk-types/openrouter"; -export class OpenRouterPlugin extends BasePlugin { - protected onEnable(): void { +class OpenRouterInstrumentationConsumer { + public register(): void { this.subscribeToOpenRouterChannels(); } - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } - private subscribeToOpenRouterChannels(): void { - this.unsubscribers.push( - traceStreamingChannel(openRouterChannels.chatSend, { - name: "openrouter.chat.send", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const chatGenerationParams = isObject(request?.chatGenerationParams) - ? request.chatGenerationParams - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle; - const { messages, ...metadata } = chatGenerationParams; - return { - input: messages, - metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), - }; - }, - extractOutput: (result) => { - return isObject(result) ? result.choices : undefined; - }, - extractMetrics: (result, startTime) => { - const metrics = parseOpenRouterMetricsFromUsage(result?.usage); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateOpenRouterChatChunks, - }), - ); - - this.unsubscribers.push( - traceAsyncChannel(openRouterChannels.embeddingsGenerate, { - name: "openrouter.embeddings.generate", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const requestBody = isObject(request?.requestBody) - ? request.requestBody - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle; - const { input, ...metadata } = requestBody; - return { - input, - metadata: buildOpenRouterEmbeddingMetadata( - metadata, - httpReferer, - xTitle, - ), - }; - }, - extractOutput: (result) => { - if (!isObject(result)) { - return undefined; - } + traceStreamingChannel(openRouterChannels.chatSend, { + name: "openrouter.chat.send", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const chatGenerationParams = isObject(request?.chatGenerationParams) + ? request.chatGenerationParams + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle; + const { messages, ...metadata } = chatGenerationParams; + return { + input: messages, + metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), + }; + }, + extractOutput: (result) => { + return isObject(result) ? result.choices : undefined; + }, + extractMetrics: (result, startTime) => { + const metrics = parseOpenRouterMetricsFromUsage(result?.usage); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateOpenRouterChatChunks, + }); - const embedding = result.data?.[0]?.embedding; - return Array.isArray(embedding) - ? { embedding_length: embedding.length } - : undefined; - }, - extractMetadata: (result) => { - if (!isObject(result)) { - return undefined; - } + traceAsyncChannel(openRouterChannels.embeddingsGenerate, { + name: "openrouter.embeddings.generate", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const requestBody = isObject(request?.requestBody) + ? request.requestBody + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle; + const { input, ...metadata } = requestBody; + return { + input, + metadata: buildOpenRouterEmbeddingMetadata( + metadata, + httpReferer, + xTitle, + ), + }; + }, + extractOutput: (result) => { + if (!isObject(result)) { + return undefined; + } - return extractOpenRouterResponseMetadata(result); - }, - extractMetrics: (result) => { - return isObject(result) - ? parseOpenRouterMetricsFromUsage(result.usage) - : {}; - }, - }), - ); + const embedding = result.data?.[0]?.embedding; + return Array.isArray(embedding) + ? { embedding_length: embedding.length } + : undefined; + }, + extractMetadata: (result) => { + if (!isObject(result)) { + return undefined; + } - this.unsubscribers.push( - traceAsyncChannel(openRouterChannels.rerankRerank, { - name: "openrouter.rerank.rerank", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const requestBody = isObject(request?.requestBody) - ? request.requestBody - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle ?? request?.appTitle; - const { documents, query, ...metadata } = requestBody; - return { - input: { - documents, - query, - }, - metadata: buildOpenRouterRerankMetadata( - metadata, - documents, - httpReferer, - xTitle, - ), - }; - }, - extractOutput: (result) => extractOpenRouterRerankOutput(result), - extractMetadata: (result) => extractOpenRouterResponseMetadata(result), - extractMetrics: (result) => - isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, - }), - ); + return extractOpenRouterResponseMetadata(result); + }, + extractMetrics: (result) => { + return isObject(result) + ? parseOpenRouterMetricsFromUsage(result.usage) + : {}; + }, + }); - this.unsubscribers.push( - traceStreamingChannel(openRouterChannels.betaResponsesSend, { - name: "openrouter.beta.responses.send", - type: SpanTypeAttribute.LLM, - extractInput: (args) => { - const request = getOpenRouterRequestArg(args); - const openResponsesRequest = isObject(request?.openResponsesRequest) - ? request.openResponsesRequest - : {}; - const httpReferer = request?.httpReferer; - const xTitle = request?.xTitle; - const { input, ...metadata } = openResponsesRequest; - return { - input, - metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), - }; - }, - extractOutput: (result) => - extractOpenRouterResponseOutput(result as Record), - extractMetadata: (result) => extractOpenRouterResponseMetadata(result), - extractMetrics: (result, startTime) => { - const metrics = parseOpenRouterMetricsFromUsage(result?.usage); - if (startTime) { - metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; - } - return metrics; - }, - aggregateChunks: aggregateOpenRouterResponseStreamEvents, - }), - ); + traceAsyncChannel(openRouterChannels.rerankRerank, { + name: "openrouter.rerank.rerank", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const requestBody = isObject(request?.requestBody) + ? request.requestBody + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle ?? request?.appTitle; + const { documents, query, ...metadata } = requestBody; + return { + input: { + documents, + query, + }, + metadata: buildOpenRouterRerankMetadata( + metadata, + documents, + httpReferer, + xTitle, + ), + }; + }, + extractOutput: (result) => extractOpenRouterRerankOutput(result), + extractMetadata: (result) => extractOpenRouterResponseMetadata(result), + extractMetrics: (result) => + isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, + }); - this.unsubscribers.push( - traceSyncStreamChannel(openRouterChannels.callModel, { - name: "openrouter.callModel", - type: SpanTypeAttribute.TASK, - extractInput: (args) => { - const request = getOpenRouterCallModelRequestArg(args); - return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: request - ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }, - }; - }, - patchResult: ({ endEvent, result, span }) => { - return patchOpenRouterCallModelResult({ - request: getOpenRouterCallModelRequestArg(endEvent.arguments), - result, - span, - }); - }, - }), - ); + traceStreamingChannel(openRouterChannels.betaResponsesSend, { + name: "openrouter.beta.responses.send", + type: SpanTypeAttribute.LLM, + extractInput: (args) => { + const request = getOpenRouterRequestArg(args); + const openResponsesRequest = isObject(request?.openResponsesRequest) + ? request.openResponsesRequest + : {}; + const httpReferer = request?.httpReferer; + const xTitle = request?.xTitle; + const { input, ...metadata } = openResponsesRequest; + return { + input, + metadata: buildOpenRouterMetadata(metadata, httpReferer, xTitle), + }; + }, + extractOutput: (result) => + extractOpenRouterResponseOutput(result as Record), + extractMetadata: (result) => extractOpenRouterResponseMetadata(result), + extractMetrics: (result, startTime) => { + const metrics = parseOpenRouterMetricsFromUsage(result?.usage); + if (startTime) { + metrics.time_to_first_token = getCurrentUnixTimestamp() - startTime; + } + return metrics; + }, + aggregateChunks: aggregateOpenRouterResponseStreamEvents, + }); - this.unsubscribers.push( - traceAsyncChannel(openRouterChannels.callModelTurn, { - name: "openrouter.beta.responses.send", - type: SpanTypeAttribute.LLM, - extractInput: (args, event) => { - const request = getOpenRouterCallModelRequestArg(args); - const metadata = request + traceSyncStreamChannel(openRouterChannels.callModel, { + name: "openrouter.callModel", + type: SpanTypeAttribute.TASK, + extractInput: (args) => { + const request = getOpenRouterCallModelRequestArg(args); + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: request ? extractOpenRouterCallModelMetadata(request) - : { provider: "openrouter" }; + : { provider: "openrouter" }, + }; + }, + patchResult: ({ endEvent, result, span }) => { + return patchOpenRouterCallModelResult({ + request: getOpenRouterCallModelRequestArg(endEvent.arguments), + result, + span, + }); + }, + }); - if (isObject(metadata) && "tools" in metadata) { - delete (metadata as Record).tools; - } + traceAsyncChannel(openRouterChannels.callModelTurn, { + name: "openrouter.beta.responses.send", + type: SpanTypeAttribute.LLM, + extractInput: (args, event) => { + const request = getOpenRouterCallModelRequestArg(args); + const metadata = request + ? extractOpenRouterCallModelMetadata(request) + : { provider: "openrouter" }; + + if (isObject(metadata) && "tools" in metadata) { + delete (metadata as Record).tools; + } + return { + input: request ? extractOpenRouterCallModelInput(request) : undefined, + metadata: { + ...metadata, + step: event.step, + step_type: event.stepType, + }, + }; + }, + extractOutput: (result) => + extractOpenRouterResponseOutput(result as Record), + extractMetadata: (result, event) => { + if (!isObject(result)) { return { - input: request - ? extractOpenRouterCallModelInput(request) - : undefined, - metadata: { - ...metadata, - step: event.step, - step_type: event.stepType, - }, + step: event?.step, + step_type: event?.stepType, }; - }, - extractOutput: (result) => - extractOpenRouterResponseOutput(result as Record), - extractMetadata: (result, event) => { - if (!isObject(result)) { - return { - step: event?.step, - step_type: event?.stepType, - }; - } + } - return { - ...(extractOpenRouterResponseMetadata(result) || {}), - ...(event?.step !== undefined ? { step: event.step } : {}), - ...(event?.stepType ? { step_type: event.stepType } : {}), - }; + return { + ...(extractOpenRouterResponseMetadata(result) || {}), + ...(event?.step !== undefined ? { step: event.step } : {}), + ...(event?.stepType ? { step_type: event.stepType } : {}), + }; + }, + extractMetrics: (result) => + isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, + }); + + traceStreamingChannel(openRouterChannels.toolExecute, { + name: "openrouter.tool", + type: SpanTypeAttribute.TOOL, + extractInput: (args, event) => ({ + input: args[0], + metadata: { + provider: "openrouter", + tool_name: event.toolName, + ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), }, - extractMetrics: (result) => - isObject(result) ? parseOpenRouterMetricsFromUsage(result.usage) : {}, }), - ); - - this.unsubscribers.push( - traceStreamingChannel(openRouterChannels.toolExecute, { - name: "openrouter.tool", - type: SpanTypeAttribute.TOOL, - extractInput: (args, event) => ({ - input: args[0], - metadata: { - provider: "openrouter", - tool_name: event.toolName, - ...(event.toolCallId ? { tool_call_id: event.toolCallId } : {}), - }, - }), - extractOutput: (result) => result, - extractMetrics: () => ({}), - aggregateChunks: (chunks) => ({ - output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, - metrics: {}, - }), + extractOutput: (result) => result, + extractMetrics: () => ({}), + aggregateChunks: (chunks) => ({ + output: chunks.length > 0 ? chunks[chunks.length - 1] : undefined, + metrics: {}, }), - ); + }); const callModelChannel = openRouterChannels.callModel.tracingChannel(); const callModelHandlers = { @@ -283,9 +260,6 @@ export class OpenRouterPlugin extends BasePlugin { }; callModelChannel.subscribe(callModelHandlers); - this.unsubscribers.push(() => { - callModelChannel.unsubscribe(callModelHandlers); - }); } } @@ -1508,3 +1482,12 @@ function normalizeError(error: unknown): Error { } export { parseOpenRouterMetricsFromUsage }; + +let openRouterInstrumentationConsumer: + | OpenRouterInstrumentationConsumer + | undefined; + +export function registerOpenRouterInstrumentation(): void { + openRouterInstrumentationConsumer ??= new OpenRouterInstrumentationConsumer(); + openRouterInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/pi-coding-agent-channels.ts b/js/src/instrumentation/providers/pi-coding-agent-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/pi-coding-agent-channels.ts rename to js/src/instrumentation/providers/pi-coding-agent-channels.ts diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts similarity index 91% rename from js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts rename to js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts index 433361974..7e7d406ed 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.test.ts +++ b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.test.ts @@ -30,7 +30,7 @@ vi.mock("../../logger", () => ({ })); import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; -import { PiCodingAgentPlugin } from "./pi-coding-agent-plugin"; +import { registerPiCodingAgentInstrumentation } from "./pi-coding-agent-instrumentation"; const PROMPT_CHANNEL = "orchestrion:@earendil-works/pi-coding-agent:AgentSession.prompt"; @@ -43,12 +43,10 @@ type TestSpan = { name?: string; }; -describe("PiCodingAgentPlugin", () => { - let plugins: PiCodingAgentPlugin[]; +describe("registerPiCodingAgentInstrumentation", () => { let spans: TestSpan[]; beforeEach(() => { - plugins = []; spans = []; interceptorsByName.clear(); mockNewTracingChannel.mockImplementation((name: string) => ({ @@ -74,25 +72,19 @@ describe("PiCodingAgentPlugin", () => { }); afterEach(() => { - for (const plugin of plugins) { - plugin.disable(); - } vi.clearAllMocks(); }); - it("registers and removes the prompt interceptor", () => { - const plugin = enablePlugin(plugins); + it("registers the prompt interceptor for the process lifetime", () => { + registerPiCodingAgentInstrumentation(); expect(interceptorsByName.has(PROMPT_CHANNEL)).toBe(true); expect(mockRemoveInterceptor).not.toHaveBeenCalled(); - - plugin.disable(); - - expect(mockRemoveInterceptor).toHaveBeenCalledTimes(1); }); it("uses interceptor ALS for prompt, LLM, and tool spans", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const finalMessage = makeAssistantMessage("done"); const originalStreamFn = vi.fn(async () => { expect(isAutoInstrumentationSuppressed()).toBe(true); @@ -200,7 +192,8 @@ describe("PiCodingAgentPlugin", () => { }); it("isolates overlapping prompts on the same agent without prompt matching", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const originalStreamFn = vi.fn(async () => makeStream(makeAssistantMessage("done")), ); @@ -258,7 +251,8 @@ describe("PiCodingAgentPlugin", () => { }); it("keeps deferred follow-up prompts open for their ALS-owned turn", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const agent = makeAgent( vi.fn(async () => makeStream(makeAssistantMessage("done"))), ); @@ -296,7 +290,8 @@ describe("PiCodingAgentPlugin", () => { }); it("preserves full-iterator stream behavior", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const message = makeAssistantMessage("done"); const { result, stream } = makeIteratorBackedStream([ { partial: message, type: "start" }, @@ -331,7 +326,8 @@ describe("PiCodingAgentPlugin", () => { }); it("forwards iterator failures and closes the underlying iterator", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const { iterator, stream } = makeIteratorBackedStream([]); iterator.next.mockRejectedValueOnce(new Error("stream next failed")); const agent = makeAgent(vi.fn(async () => stream)); @@ -364,7 +360,8 @@ describe("PiCodingAgentPlugin", () => { }); it("closes active prompt and LLM spans when the target rejects", async () => { - const interceptor = promptInterceptor(enablePlugin(plugins)); + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const agent = makeAgent( vi.fn(async () => makeStream(makeAssistantMessage("unused"))), ); @@ -396,9 +393,9 @@ describe("PiCodingAgentPlugin", () => { expect(taskSpan?.end).toHaveBeenCalledTimes(1); }); - it("ends in-flight prompts on disable while leaving the safe patch installed", async () => { - const plugin = enablePlugin(plugins); - const interceptor = promptInterceptor(plugin); + it("keeps in-flight prompts active until the call completes", async () => { + registerPiCodingAgentInstrumentation(); + const interceptor = promptInterceptor(); const originalStreamFn = vi.fn(async () => makeStream(makeAssistantMessage("done")), ); @@ -416,12 +413,10 @@ describe("PiCodingAgentPlugin", () => { ); const patchedStreamFn = agent.streamFn; - plugin.disable(); - const taskSpan = findSpan(spans, "AgentSession.prompt"); expect(agent.streamFn).toBe(patchedStreamFn); expect(agent.streamFn).not.toBe(originalStreamFn); - expect(taskSpan?.end).toHaveBeenCalledTimes(1); + expect(taskSpan?.end).not.toHaveBeenCalled(); targetGate.resolve(); await result; @@ -429,14 +424,7 @@ describe("PiCodingAgentPlugin", () => { }); }); -function enablePlugin(plugins: PiCodingAgentPlugin[]): PiCodingAgentPlugin { - const plugin = new PiCodingAgentPlugin(); - plugins.push(plugin); - plugin.enable(); - return plugin; -} - -function promptInterceptor(_plugin: PiCodingAgentPlugin): any { +function promptInterceptor(): any { const interceptor = interceptorsByName.get(PROMPT_CHANNEL); expect(interceptor).toEqual(expect.any(Function)); return interceptor; diff --git a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts similarity index 93% rename from js/src/instrumentation/plugins/pi-coding-agent-plugin.ts rename to js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts index da6c50b30..11829e004 100644 --- a/js/src/instrumentation/plugins/pi-coding-agent-plugin.ts +++ b/js/src/instrumentation/providers/pi-coding-agent-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import iso, { type IsoAsyncLocalStorage } from "../../isomorph"; import { debugLogger } from "../../debug-logger"; @@ -44,7 +44,6 @@ type PiPromptState = { deferCompletionUntilTurnEnd: boolean; finalized: boolean; metrics: Record; - onFinalize?: (state: PiPromptState) => void; metadata: Record; output?: unknown; promptCallEnded: boolean; @@ -77,74 +76,52 @@ let piPromptContextStore: | IsoAsyncLocalStorage | undefined; -export class PiCodingAgentPlugin extends BasePlugin { - private readonly activePromptStates = new Set(); - - protected onEnable(): void { +class PiCodingAgentInstrumentationConsumer { + public register(): void { this.interceptPrompt(); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - - for (const state of [...this.activePromptStates]) { - finishPiPromptRun(state); - } - } - private interceptPrompt(): void { - this.unsubscribers.push( - piCodingAgentChannels.prompt.intercept( - (target, thisArg, args, additional) => { - const invokeTarget = () => Reflect.apply(target, thisArg, args); - let state: PiPromptState | undefined; - try { - state = startPiPromptRun( - { - ...additional, - arguments: args, - self: thisArg, - }, - (finalizedState) => { - this.activePromptStates.delete(finalizedState); - }, - ); - } catch (error) { - logInstrumentationError("Pi Coding Agent prompt start", error); - } + void piCodingAgentChannels.prompt.intercept( + (target, thisArg, args, additional) => { + const invokeTarget = () => Reflect.apply(target, thisArg, args); + let state: PiPromptState | undefined; + try { + state = startPiPromptRun({ + ...additional, + arguments: args, + self: thisArg, + }); + } catch (error) { + logInstrumentationError("Pi Coding Agent prompt start", error); + } - if (!state) { - return runWithAutoInstrumentationSuppressed(invokeTarget); - } - this.activePromptStates.add(state); + if (!state) { + return runWithAutoInstrumentationSuppressed(invokeTarget); + } + return promptContextStore().run(state, () => + runWithAutoInstrumentationSuppressed(() => { + let result: PromiseLike; + try { + result = invokeTarget(); + } catch (error) { + finishPiPromptRun(state, error); + throw error; + } - return promptContextStore().run(state, () => - runWithAutoInstrumentationSuppressed(() => { - let result: PromiseLike; - try { - result = invokeTarget(); - } catch (error) { + return Promise.resolve(result).then( + (value) => { + finishPiPromptCall(state); + return value; + }, + (error) => { finishPiPromptRun(state, error); throw error; - } - - return Promise.resolve(result).then( - (value) => { - finishPiPromptCall(state); - return value; - }, - (error) => { - finishPiPromptRun(state, error); - throw error; - }, - ); - }), - ); - }, - ), + }, + ); + }), + ); + }, ); } } @@ -159,7 +136,6 @@ function finishPiPromptCall(state: PiPromptState): void { function startPiPromptRun( event: ChannelMessage, - onFinalize?: (state: PiPromptState) => void, ): PiPromptState | undefined { const session = extractSession(event); const agent = session?.agent; @@ -206,7 +182,6 @@ function startPiPromptRun( finalized: false, metadata, metrics: {}, - onFinalize, promptCallEnded: false, span, startTime: getCurrentUnixTimestamp(), @@ -595,7 +570,6 @@ function finishPiPromptRun(state: PiPromptState, error?: unknown): void { return; } state.finalized = true; - state.onFinalize?.(state); finishOpenLlmSpans(state, error); finishOpenToolSpans(state, error); @@ -1085,3 +1059,13 @@ function safeLog(span: Span, event: Parameters[0]): void { function logInstrumentationError(context: string, error: unknown): void { debugLogger.debug(`${context}:`, error); } + +let piCodingAgentInstrumentationConsumer: + | PiCodingAgentInstrumentationConsumer + | undefined; + +export function registerPiCodingAgentInstrumentation(): void { + piCodingAgentInstrumentationConsumer ??= + new PiCodingAgentInstrumentationConsumer(); + piCodingAgentInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-channels.ts b/js/src/instrumentation/providers/strands-agent-sdk-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/strands-agent-sdk-channels.ts rename to js/src/instrumentation/providers/strands-agent-sdk-channels.ts diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.test.ts similarity index 95% rename from js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts rename to js/src/instrumentation/providers/strands-agent-sdk-instrumentation.test.ts index 54340d1a3..5c0986ca8 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.test.ts +++ b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.test.ts @@ -5,7 +5,6 @@ const { mockWithCurrent, mockNewAsyncLocalStorage, mockStartSpan, - mockUnbindStore, } = vi.hoisted(() => ({ mockBindStore: vi.fn(), mockWithCurrent: vi.fn(), @@ -28,7 +27,6 @@ const { }; }), mockStartSpan: vi.fn(), - mockUnbindStore: vi.fn(), })); vi.mock("../../isomorph", () => ({ @@ -51,11 +49,11 @@ vi.mock("../../logger", async (importOriginal) => { import iso from "../../isomorph"; import { Attachment } from "../../logger"; import { isAutoInstrumentationSuppressed } from "../auto-instrumentation-suppression"; -import { StrandsAgentSDKPlugin } from "./strands-agent-sdk-plugin"; +import { registerStrandsAgentSDKInstrumentation } from "./strands-agent-sdk-instrumentation"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("StrandsAgentSDKPlugin", () => { +describe("registerStrandsAgentSDKInstrumentation", () => { let handlersByName: Map; let spans: Array<{ args: any; @@ -77,11 +75,9 @@ describe("StrandsAgentSDKPlugin", () => { mockNewTracingChannel.mockImplementation((name: string) => ({ start: { bindStore: mockBindStore, - unbindStore: mockUnbindStore, }, subscribe: vi.fn((handlers) => handlersByName.set(name, handlers)), traceSync: vi.fn((fn) => fn()), - unsubscribe: vi.fn(), })); currentSpan = undefined; mockWithCurrent.mockImplementation((span: any, callback: () => unknown) => { @@ -123,8 +119,7 @@ describe("StrandsAgentSDKPlugin", () => { }); it("subscribes to Strands stream channels and binds suppression", () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); expect( handlersByName.has("orchestrion:@strands-agents/sdk:Agent.stream"), @@ -136,15 +131,10 @@ describe("StrandsAgentSDKPlugin", () => { handlersByName.has("orchestrion:@strands-agents/sdk:Swarm.stream"), ).toBe(true); expect(mockBindStore).toHaveBeenCalledTimes(3); - - plugin.disable(); - - expect(mockUnbindStore).toHaveBeenCalledTimes(3); }); it("records agent model and tool spans from stream events", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const handlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Agent.stream", @@ -292,8 +282,7 @@ describe("StrandsAgentSDKPlugin", () => { ])( "converts media from %s to one attachment shared by agent and model spans", async (_description, bytes) => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const handlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Agent.stream", @@ -353,8 +342,7 @@ describe("StrandsAgentSDKPlugin", () => { ); it("parents nested agent spans under active graph nodes", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const graphHandlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Graph.stream", @@ -441,8 +429,7 @@ describe("StrandsAgentSDKPlugin", () => { }); it("parents nested graph spans under active graph nodes", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const graphHandlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Graph.stream", @@ -530,8 +517,7 @@ describe("StrandsAgentSDKPlugin", () => { }); it("ends open child spans when a stream errors", async () => { - const plugin = new StrandsAgentSDKPlugin(); - plugin.enable(); + registerStrandsAgentSDKInstrumentation(); const handlers = handlersByName.get( "orchestrion:@strands-agents/sdk:Agent.stream", diff --git a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.ts similarity index 97% rename from js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts rename to js/src/instrumentation/providers/strands-agent-sdk-instrumentation.ts index 255578a13..5142f2d5c 100644 --- a/js/src/instrumentation/plugins/strands-agent-sdk-plugin.ts +++ b/js/src/instrumentation/providers/strands-agent-sdk-instrumentation.ts @@ -1,4 +1,4 @@ -import { BasePlugin, toLoggedError } from "../core"; +import { toLoggedError } from "../core"; import type { ChannelMessage } from "../core/channel-definitions"; import { isAsyncIterable, patchStreamIfNeeded } from "../core/stream-patcher"; import type { IsoChannelHandlers } from "../../isomorph"; @@ -101,10 +101,10 @@ type StrandsAttachmentCache = { strings: LRUCache>; }; -export class StrandsAgentSDKPlugin extends BasePlugin { +class StrandsAgentSDKInstrumentationConsumer { private readonly activeChildParents: ActiveChildParents = new WeakMap(); - protected onEnable(): void { + public register(): void { this.subscribeToAgentStream(); this.subscribeToMultiAgentStream( strandsAgentSDKChannels.graphStream, @@ -116,18 +116,10 @@ export class StrandsAgentSDKPlugin extends BasePlugin { ); } - protected onDisable(): void { - for (const unsubscribe of this.unsubscribers) { - unsubscribe(); - } - this.unsubscribers = []; - } - private subscribeToAgentStream(): void { const channel = strandsAgentSDKChannels.agentStream.tracingChannel(); const states = new WeakMap(); - const unbindAutoInstrumentationSuppression = - bindAutoInstrumentationSuppressionToStart(channel); + bindAutoInstrumentationSuppressionToStart(channel); const handlers: IsoChannelHandlers< ChannelMessage @@ -176,10 +168,6 @@ export class StrandsAgentSDKPlugin extends BasePlugin { }; channel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindAutoInstrumentationSuppression?.(); - channel.unsubscribe(handlers); - }); } private subscribeToMultiAgentStream( @@ -188,8 +176,7 @@ export class StrandsAgentSDKPlugin extends BasePlugin { ): void { const tracingChannel = channel.tracingChannel(); const states = new WeakMap(); - const unbindAutoInstrumentationSuppression = - bindAutoInstrumentationSuppressionToStart(tracingChannel); + bindAutoInstrumentationSuppressionToStart(tracingChannel); const handlers: IsoChannelHandlers> = { start: (event) => { @@ -250,10 +237,6 @@ export class StrandsAgentSDKPlugin extends BasePlugin { }; tracingChannel.subscribe(handlers); - this.unsubscribers.push(() => { - unbindAutoInstrumentationSuppression?.(); - tracingChannel.unsubscribe(handlers); - }); } } @@ -1393,3 +1376,13 @@ function safeLog(span: Span, event: Parameters[0]): void { function logInstrumentationError(context: string, error: unknown): void { debugLogger.debug(`${context}:`, error); } + +let strandsAgentSDKInstrumentationConsumer: + | StrandsAgentSDKInstrumentationConsumer + | undefined; + +export function registerStrandsAgentSDKInstrumentation(): void { + strandsAgentSDKInstrumentationConsumer ??= + new StrandsAgentSDKInstrumentationConsumer(); + strandsAgentSDKInstrumentationConsumer.register(); +} diff --git a/js/src/instrumentation/plugins/voyageai-channels.ts b/js/src/instrumentation/providers/voyageai-channels.ts similarity index 100% rename from js/src/instrumentation/plugins/voyageai-channels.ts rename to js/src/instrumentation/providers/voyageai-channels.ts diff --git a/js/src/instrumentation/plugins/voyageai-plugin.test.ts b/js/src/instrumentation/providers/voyageai-instrumentation.test.ts similarity index 98% rename from js/src/instrumentation/plugins/voyageai-plugin.test.ts rename to js/src/instrumentation/providers/voyageai-instrumentation.test.ts index 65cfca93c..2cdb491b4 100644 --- a/js/src/instrumentation/plugins/voyageai-plugin.test.ts +++ b/js/src/instrumentation/providers/voyageai-instrumentation.test.ts @@ -9,7 +9,7 @@ try { // Best-effort initialization for test environments. } -describe("VoyageAIPlugin", () => { +describe("registerVoyageAIInstrumentation", () => { let backgroundLogger: ReturnType< typeof _exportsForTestingOnly.useTestBackgroundLogger >; @@ -21,7 +21,7 @@ describe("VoyageAIPlugin", () => { beforeEach(() => { backgroundLogger = _exportsForTestingOnly.useTestBackgroundLogger(); initLogger({ - projectName: "voyageai-plugin.test.ts", + projectName: "voyageai-instrumentation.test.ts", projectId: "test-project-id", }); }); diff --git a/js/src/instrumentation/plugins/voyageai-plugin.ts b/js/src/instrumentation/providers/voyageai-instrumentation.ts similarity index 89% rename from js/src/instrumentation/plugins/voyageai-plugin.ts rename to js/src/instrumentation/providers/voyageai-instrumentation.ts index 37f864b26..ac2304e81 100644 --- a/js/src/instrumentation/plugins/voyageai-plugin.ts +++ b/js/src/instrumentation/providers/voyageai-instrumentation.ts @@ -17,8 +17,6 @@ import { isAutoInstrumentationSuppressed, runWithAutoInstrumentationSuppressed, } from "../auto-instrumentation-suppression"; -import { BasePlugin } from "../core"; -import { unsubscribeAll } from "../core/channel-tracing"; import { voyageAIChannels } from "./voyageai-channels"; const RERANK_METADATA_ALLOWLIST = new Set([ @@ -28,42 +26,34 @@ const RERANK_METADATA_ALLOWLIST = new Set([ "truncation", ]); -export class VoyageAIPlugin extends BasePlugin { - protected onEnable(): void { - this.unsubscribers.push( - interceptVoyageAICall( - voyageAIChannels.embed, - "voyageai.embed", - extractTextEmbeddingInput, - summarizeEmbeddingOutput, - extractEmbeddingUsageMetrics, - ), - interceptVoyageAICall( - voyageAIChannels.multimodalEmbed, - "voyageai.multimodalEmbed", - extractMultimodalEmbeddingInput, - summarizeEmbeddingOutput, - extractEmbeddingUsageMetrics, - ), - interceptVoyageAICall( - voyageAIChannels.rerank, - "voyageai.rerank", - extractRerankInput, - summarizeRerankOutput, - ), - interceptVoyageAICall( - voyageAIChannels.contextualizedEmbed, - "voyageai.contextualizedEmbed", - extractContextualizedEmbeddingInput, - summarizeContextualizedEmbeddingOutput, - extractEmbeddingUsageMetrics, - ), - ); - } - - protected onDisable(): void { - this.unsubscribers = unsubscribeAll(this.unsubscribers); - } +export function registerVoyageAIInstrumentation(): void { + interceptVoyageAICall( + voyageAIChannels.embed, + "voyageai.embed", + extractTextEmbeddingInput, + summarizeEmbeddingOutput, + extractEmbeddingUsageMetrics, + ); + interceptVoyageAICall( + voyageAIChannels.multimodalEmbed, + "voyageai.multimodalEmbed", + extractMultimodalEmbeddingInput, + summarizeEmbeddingOutput, + extractEmbeddingUsageMetrics, + ); + interceptVoyageAICall( + voyageAIChannels.rerank, + "voyageai.rerank", + extractRerankInput, + summarizeRerankOutput, + ); + interceptVoyageAICall( + voyageAIChannels.contextualizedEmbed, + "voyageai.contextualizedEmbed", + extractContextualizedEmbeddingInput, + summarizeContextualizedEmbeddingOutput, + extractEmbeddingUsageMetrics, + ); } type VoyageAIResult = @@ -455,13 +445,10 @@ function extractUsageMetrics(result: VoyageAIResult): Record { const rawResponse = isObject(result.rawResponse) ? result.rawResponse : undefined; - const usage = ( - isObject(result.usage) - ? result.usage - : isObject(rawResponse?.usage) - ? rawResponse.usage - : undefined - ) as VoyageAIUsage | undefined; + const usageValue = isObject(result.usage) ? result.usage : rawResponse?.usage; + const usage = isObject(usageValue) + ? (usageValue as VoyageAIUsage) + : undefined; const tokens = typeof result.totalTokens === "number" ? result.totalTokens diff --git a/js/src/instrumentation/registry.test.ts b/js/src/instrumentation/registry.test.ts index fdd49a5c2..97d325cea 100644 --- a/js/src/instrumentation/registry.test.ts +++ b/js/src/instrumentation/registry.test.ts @@ -19,32 +19,25 @@ import iso from "../isomorph"; const mockNewTracingChannel = iso.newTracingChannel as ReturnType; -describe("Plugin Registry", () => { +describe("Instrumentation Registry", () => { beforeEach(() => { // Setup mock channel const mockChannel = { subscribe: vi.fn(), - unsubscribe: vi.fn(), hasSubscribers: false, }; mockNewTracingChannel.mockReturnValue(mockChannel); }); - // Clean up after each test - afterEach(() => { - registry.disable(); - }); - it("should not be enabled by default", () => { const testRegistry = new (registry.constructor as any)(); expect(testRegistry.isEnabled()).toBe(false); }); - it("should enable plugins when enable() is called", () => { + it("should enable instrumentation consumers when enable() is called", () => { const testRegistry = new (registry.constructor as any)(); testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should be idempotent (calling enable() multiple times)", () => { @@ -52,12 +45,11 @@ describe("Plugin Registry", () => { testRegistry.enable(); testRegistry.enable(); // Should not throw expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should block a second instance from subscribing when another is already enabled", () => { // Regression test for BT-5139: when the SDK is loaded from two different - // module paths in the same process, each gets its own PluginRegistry + // module paths in the same process, each gets its own instrumentation registry // instance. Without cross-instance deduplication, both would subscribe to // the same global hook, causing every OpenAI call to produce two // LLM spans. @@ -80,8 +72,6 @@ describe("Plugin Registry", () => { instanceB.enable(); expect(instanceB.isEnabled()).toBe(false); } finally { - instanceA.disable(); - instanceB.disable(); delete (globalThis as any)[stateKey]; } }); @@ -97,7 +87,6 @@ describe("Plugin Registry", () => { testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); } finally { - testRegistry.disable(); delete (globalThis as any)[stateKey]; } }); @@ -116,7 +105,6 @@ describe("Plugin Registry", () => { expect(warnSpy[0]).toContain("Cannot configure instrumentation"); } finally { console.warn = originalWarn; - testRegistry.disable(); } }); @@ -133,33 +121,11 @@ describe("Plugin Registry", () => { expect(warnSpy.length).toBe(0); } finally { console.warn = originalWarn; - testRegistry.disable(); } }); - - it("should disable plugins when disable() is called", () => { - const testRegistry = new (registry.constructor as any)(); - testRegistry.enable(); - expect(testRegistry.isEnabled()).toBe(true); - - testRegistry.disable(); - expect(testRegistry.isEnabled()).toBe(false); - }); - - it("should be idempotent (calling disable() multiple times)", () => { - const testRegistry = new (registry.constructor as any)(); - testRegistry.enable(); - testRegistry.disable(); - testRegistry.disable(); // Should not throw - expect(testRegistry.isEnabled()).toBe(false); - }); }); describe("configureInstrumentation API", () => { - afterEach(() => { - registry.disable(); - }); - it("should export configureInstrumentation function", () => { expect(typeof configureInstrumentation).toBe("function"); }); @@ -195,7 +161,6 @@ describe("Environment Variable Configuration", () => { afterEach(async () => { const iso = (await import("../isomorph")).default; iso.getEnv = originalGetEnv; - registry.disable(); }); it("should parse BRAINTRUST_DISABLE_INSTRUMENTATION with single SDK", async () => { @@ -212,7 +177,6 @@ describe("Environment Variable Configuration", () => { // OpenAI should be disabled, others enabled by default expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should parse BRAINTRUST_DISABLE_INSTRUMENTATION with multiple SDKs", async () => { @@ -229,7 +193,6 @@ describe("Environment Variable Configuration", () => { // Both should be disabled expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should handle whitespace in BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { @@ -245,7 +208,6 @@ describe("Environment Variable Configuration", () => { testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should handle empty BRAINTRUST_DISABLE_INSTRUMENTATION", async () => { @@ -262,7 +224,6 @@ describe("Environment Variable Configuration", () => { // All should be enabled (nothing disabled) expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); it("should be case-insensitive for SDK names", async () => { @@ -278,6 +239,5 @@ describe("Environment Variable Configuration", () => { testRegistry.enable(); expect(testRegistry.isEnabled()).toBe(true); - testRegistry.disable(); }); }); diff --git a/js/src/instrumentation/registry.ts b/js/src/instrumentation/registry.ts index 0ed195ae8..010aee634 100644 --- a/js/src/instrumentation/registry.ts +++ b/js/src/instrumentation/registry.ts @@ -1,11 +1,11 @@ /** - * Plugin registry and configuration for auto-instrumentation. + * Registry and configuration for auto-instrumentation. * - * Plugins are automatically enabled when the Braintrust library is loaded. + * Instrumentation consumers are automatically enabled when Braintrust loads. * Users can disable specific integrations programmatically or via environment variables. */ -import { BraintrustPlugin } from "./braintrust-plugin"; +import { registerInstrumentationConsumers } from "./instrumentation-consumers"; import iso from "../isomorph"; import { getDefaultInstrumentationIntegrations, @@ -16,7 +16,7 @@ import { GLOBAL_INSTRUMENTATION_HOOKS_PROTOCOL_VERSION } from "../global-instrum export type { InstrumentationConfig } from "./config"; -// Key used to stamp the active PluginRegistry instance onto the shared +// Key used to stamp the active InstrumentationRegistry instance onto the shared // braintrust state object (globalThis[Symbol.for("braintrust-state")]). // // The braintrust state is already shared across all SDK instances loaded in @@ -43,8 +43,7 @@ function getSharedState(): Record | undefined { : undefined; } -class PluginRegistry { - private braintrustPlugin: BraintrustPlugin | null = null; +class InstrumentationRegistry { private config: InstrumentationConfig = {}; private enabled = false; @@ -65,7 +64,7 @@ class PluginRegistry { } /** - * Enable all configured plugins. + * Enable all configured instrumentation consumers. * Called automatically when the library is loaded. */ enable(): void { @@ -73,7 +72,7 @@ class PluginRegistry { return; } - // If another SDK instance in the same process already registered plugins, + // If another SDK instance already registered instrumentation consumers, // skip to avoid duplicate global hook subscriptions. const sharedState = getSharedState(); if (sharedState) { @@ -95,31 +94,8 @@ class PluginRegistry { }, }; - // Enable BraintrustPlugin with configuration - this.braintrustPlugin = new BraintrustPlugin(finalConfig); - this.braintrustPlugin.enable(); - } - - /** - * Disable all plugins. - * Primarily used for testing. - */ - disable(): void { - if (!this.enabled) { - return; - } - - this.enabled = false; - - const sharedState = getSharedState(); - if (sharedState && sharedState[REGISTRY_STATE_KEY] === this) { - delete sharedState[REGISTRY_STATE_KEY]; - } - - if (this.braintrustPlugin) { - this.braintrustPlugin.disable(); - this.braintrustPlugin = null; - } + // Enable the configured instrumentation consumers. + registerInstrumentationConsumers(finalConfig); } /** @@ -148,9 +124,9 @@ class PluginRegistry { } /** - * Global plugin registry instance. + * Global instrumentation registry instance. */ -export const registry = new PluginRegistry(); +export const registry = new InstrumentationRegistry(); /** * Configure auto-instrumentation. diff --git a/js/src/isomorph.ts b/js/src/isomorph.ts index d23ddcc7e..0b500bcec 100644 --- a/js/src/isomorph.ts +++ b/js/src/isomorph.ts @@ -1,7 +1,7 @@ -import { - type GitMetadataSettingsType as GitMetadataSettings, - type RepoInfoType as RepoInfo, -} from "./generated_types"; +import type { + GitMetadataSettingsType as GitMetadataSettings, + RepoInfoType as RepoInfo, +} from "./generated_plain_types"; import { newGlobalTracingChannel, type GlobalHookAsyncLocalStorage, diff --git a/js/src/logger.ts b/js/src/logger.ts index 9733c3030..fcf88d9a4 100644 --- a/js/src/logger.ts +++ b/js/src/logger.ts @@ -66,37 +66,38 @@ import { getObjValueByPath, } from "./util"; import { - type AnyModelParamsType as AnyModelParam, AttachmentReference as attachmentReferenceSchema, - type AttachmentReferenceType as AttachmentReference, BraintrustAttachmentReference as BraintrustAttachmentReferenceSchema, - type BraintrustAttachmentReferenceType as BraintrustAttachmentReference, BraintrustModelParams as braintrustModelParamsSchema, ChatCompletionTool as chatCompletionToolSchema, - type ChatCompletionToolType as ChatCompletionTool, ExternalAttachmentReference as ExternalAttachmentReferenceSchema, - type ExternalAttachmentReferenceType as ExternalAttachmentReference, - type ModelParamsType as ModelParams, ResponseFormatJsonSchema as responseFormatJsonSchemaSchema, AttachmentStatus as attachmentStatusSchema, - type AttachmentStatusType as AttachmentStatus, GitMetadataSettings as gitMetadataSettingsSchema, - type GitMetadataSettingsType as GitMetadataSettings, - type ChatCompletionMessageParamType as Message, - type ChatCompletionOpenAIMessageParamType as OpenAIMessage, DatasetSnapshot as datasetSnapshotSchema, - type DatasetSnapshotType as DatasetSnapshot, PromptData as promptDataSchema, - type PromptDataType as PromptData, Prompt as promptSchema, - type PromptType as PromptRow, - type PromptSessionEventType as PromptSessionEvent, - type RepoInfoType as RepoInfo, - type ObjectReferenceType as ObjectReference, - type PromptBlockDataType as PromptBlockData, - type ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, - type ObjectReferenceType, } from "./generated_types"; +import type { + AnyModelParamsType as AnyModelParam, + AttachmentReferenceType as AttachmentReference, + BraintrustAttachmentReferenceType as BraintrustAttachmentReference, + ChatCompletionToolType as ChatCompletionTool, + ExternalAttachmentReferenceType as ExternalAttachmentReference, + ModelParamsType as ModelParams, + AttachmentStatusType as AttachmentStatus, + GitMetadataSettingsType as GitMetadataSettings, + ChatCompletionMessageParamType as Message, + ChatCompletionOpenAIMessageParamType as OpenAIMessage, + DatasetSnapshotType as DatasetSnapshot, + PromptDataType as PromptData, + PromptType as PromptRow, + PromptSessionEventType as PromptSessionEvent, + RepoInfoType as RepoInfo, + ObjectReferenceType as ObjectReference, + PromptBlockDataType as PromptBlockData, + ResponseFormatJsonSchemaType as ResponseFormatJsonSchema, +} from "./generated_plain_types"; const BRAINTRUST_ATTACHMENT = BraintrustAttachmentReferenceSchema.shape.type.value; @@ -120,16 +121,21 @@ const datasetRestorePreviewResultSchema = z.object({ rows_to_restore: z.number(), rows_to_delete: z.number(), }); -export type DatasetRestorePreviewResult = z.infer< - typeof datasetRestorePreviewResultSchema ->; +export type DatasetRestorePreviewResult = { + rows_to_restore: number; + rows_to_delete: number; +}; const datasetRestoreResultSchema = z.object({ xact_id: z.string().nullable(), rows_restored: z.number(), rows_deleted: z.number(), }); -export type DatasetRestoreResult = z.infer; +export type DatasetRestoreResult = { + xact_id: string | null; + rows_restored: number; + rows_deleted: number; +}; const parametersRowSchema = z.object({ id: z.string().uuid(), @@ -148,7 +154,21 @@ const parametersRowSchema = z.object({ .union([z.object({}).partial().passthrough(), z.null()]) .optional(), }); -type ParametersRow = z.infer; +type ParametersRow = { + id: string; + _xact_id: string; + project_id: string; + name: string; + slug: string; + description?: string | null; + function_type: "parameters"; + function_data: { + type: "parameters"; + data?: Record; + __schema: Record; + }; + metadata?: Record | null; +}; import { waitUntil } from "@vercel/functions"; import { @@ -714,7 +734,18 @@ const loginSchema = z.strictObject({ debugLogLevelDisabled: z.boolean().optional(), }); -export type SerializedBraintrustState = z.infer; +export type SerializedBraintrustState = { + appUrl: string; + appPublicUrl: string; + orgName: string; + apiUrl: string; + proxyUrl: string; + loginToken: string; + orgId?: string | null; + gitMetadataSettings?: GitMetadataSettings | null; + debugLogLevel?: "error" | "warn" | "info" | "debug"; + debugLogLevelDisabled?: boolean; +}; let stateNonce = 0; @@ -1863,7 +1894,10 @@ const attachmentMetadataSchema = z.object({ status: attachmentStatusSchema, }); -type AttachmentMetadata = z.infer; +type AttachmentMetadata = { + downloadUrl: string; + status: AttachmentStatus; +}; /** * A readonly alternative to `Attachment`, which can be used for fetching @@ -2784,14 +2818,22 @@ function castLogger( return logger as unknown as Logger; } -export const logs3OverflowUploadSchema = z.object({ - method: z.enum(["PUT", "POST"]), - signedUrl: z.string().url(), - headers: z.record(z.string()).optional(), - fields: z.record(z.string()).optional(), - key: z.string().min(1), -}); -export type Logs3OverflowUpload = z.infer; +export type Logs3OverflowUpload = { + method: "PUT" | "POST"; + signedUrl: string; + headers?: Record; + fields?: Record; + key: string; +}; + +export const logs3OverflowUploadSchema: z.ZodType = + z.object({ + method: z.enum(["PUT", "POST"]), + signedUrl: z.string().url(), + headers: z.record(z.string()).optional(), + fields: z.record(z.string()).optional(), + key: z.string().min(1), + }); export type Logs3OverflowInputRow = { object_ids: Record; @@ -8375,7 +8417,7 @@ export class Dataset< metadata?: Record; tags?: string[]; output?: unknown; - origin?: ObjectReferenceType; + origin?: ObjectReference; isMerge?: boolean; }): LazyValue { return new LazyValue(async () => { @@ -8434,7 +8476,7 @@ export class Dataset< readonly metadata?: Record; readonly id?: string; readonly output?: unknown; - readonly origin?: ObjectReferenceType; + readonly origin?: ObjectReference; }): string { this.validateEvent({ metadata, expected, output, tags }); diff --git a/js/src/node/index.ts b/js/src/node/index.ts index ba75fa586..60e1efe09 100644 --- a/js/src/node/index.ts +++ b/js/src/node/index.ts @@ -42,10 +42,10 @@ * }); * ``` * - * Finally, run the script with `npx braintrust eval hello.eval.ts`. + * Finally, run the script with `npx bt eval hello.eval.ts`. * * ```bash - * BRAINTRUST_API_KEY= npx braintrust eval hello.eval.ts + * BRAINTRUST_API_KEY= npx bt eval hello.eval.ts * ``` * * @module braintrust diff --git a/js/src/parameters.test.ts b/js/src/parameters.test.ts index b22953561..69576dd9a 100644 --- a/js/src/parameters.test.ts +++ b/js/src/parameters.test.ts @@ -35,7 +35,6 @@ test("parameters are passed to task", async () => { }, }, new NoopProgressReporter(), - [], undefined, { prefix: "start:", suffix: ":end" }, true, @@ -81,7 +80,6 @@ test("prompt parameter is passed correctly", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -163,7 +161,6 @@ test("custom parameter values override defaults", async () => { }, }, new NoopProgressReporter(), - [], undefined, { prefix: "custom:", @@ -195,7 +192,6 @@ test("array parameter is handled correctly", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -234,7 +230,6 @@ test("object parameter is handled correctly", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -265,7 +260,6 @@ test("model parameter defaults to configured value", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, @@ -293,7 +287,6 @@ test("model parameter is required when default is missing", async () => { }, }, new NoopProgressReporter(), - [], undefined, undefined, true, diff --git a/js/src/prompt-schemas.ts b/js/src/prompt-schemas.ts index c4203f90e..46ad5f307 100644 --- a/js/src/prompt-schemas.ts +++ b/js/src/prompt-schemas.ts @@ -1,15 +1,23 @@ import { z } from "zod/v3"; import { ToolFunctionDefinition as toolFunctionDefinitionSchema, - type ToolFunctionDefinitionType as ToolFunctionDefinition, ChatCompletionMessageParam as chatCompletionMessageParamSchema, ModelParams as modelParamsSchema, - type PromptBlockDataType as PromptBlockData, - type PromptDataType as PromptData, } from "./generated_types"; +import type { + ToolFunctionDefinitionType as ToolFunctionDefinition, + ChatCompletionMessageParamType, + ModelParamsType, + PromptBlockDataType as PromptBlockData, + PromptDataType as PromptData, +} from "./generated_plain_types"; // This roughly maps to promptBlockDataSchema, but is more ergonomic for the user. -export const promptContentsSchema = z.union([ +export type PromptContents = + | { prompt: string } + | { messages: ChatCompletionMessageParamType[] }; + +const internalPromptContentsSchema = z.union([ z.object({ prompt: z.string(), }), @@ -17,10 +25,20 @@ export const promptContentsSchema = z.union([ messages: z.array(chatCompletionMessageParamSchema), }), ]); +export const promptContentsSchema: z.ZodType< + PromptContents, + z.ZodTypeDef, + unknown +> = internalPromptContentsSchema; -export type PromptContents = z.infer; +export type PromptDefinition = PromptContents & { + model: string; + params?: ModelParamsType; + templateFormat?: "mustache" | "nunjucks" | "none"; + environments?: string[]; +}; -export const promptDefinitionSchema = promptContentsSchema.and( +const internalPromptDefinitionSchema = internalPromptContentsSchema.and( z.object({ model: z.string(), params: modelParamsSchema.optional(), @@ -28,18 +46,27 @@ export const promptDefinitionSchema = promptContentsSchema.and( environments: z.array(z.string()).optional(), }), ); +export const promptDefinitionSchema: z.ZodType< + PromptDefinition, + z.ZodTypeDef, + unknown +> = internalPromptDefinitionSchema; -export type PromptDefinition = z.infer; - -export const promptDefinitionWithToolsSchema = promptDefinitionSchema.and( - z.object({ - tools: z.array(toolFunctionDefinitionSchema).optional(), - }), -); +export type PromptDefinitionWithTools = PromptDefinition & { + tools?: ToolFunctionDefinition[]; +}; -export type PromptDefinitionWithTools = z.infer< - typeof promptDefinitionWithToolsSchema ->; +const internalPromptDefinitionWithToolsSchema = + internalPromptDefinitionSchema.and( + z.object({ + tools: z.array(toolFunctionDefinitionSchema).optional(), + }), + ); +export const promptDefinitionWithToolsSchema: z.ZodType< + PromptDefinitionWithTools, + z.ZodTypeDef, + unknown +> = internalPromptDefinitionWithToolsSchema; export function promptDefinitionToPromptData( promptDefinition: PromptDefinition, diff --git a/js/src/public-types.test.ts b/js/src/public-types.test.ts new file mode 100644 index 000000000..0583994df --- /dev/null +++ b/js/src/public-types.test.ts @@ -0,0 +1,69 @@ +import { expectTypeOf, test } from "vitest"; +import { z } from "zod/v3"; + +import { + AttachmentReference, + braintrustStreamChunkSchema, + logs3OverflowUploadSchema, + promptContentsSchema, + promptDefinitionSchema, + promptDefinitionWithToolsSchema, + type BraintrustStreamChunk, + type EvalParameters, + type Logs3OverflowUpload, + type PromptContents, + type PromptDefinition, + type PromptDefinitionWithTools, +} from "./exports"; +import type { InferParameters } from "./eval-parameters"; +import type { AttachmentReferenceType } from "./generated_plain_types"; +import type { Prompt } from "./logger"; +import { + spanComponentsV3Schema, + type SpanComponentsV3Data, +} from "../util/span_identifier_v3"; +import { + spanComponentsV4Schema, + type SpanComponentsV4Data, +} from "../util/span_identifier_v4"; + +test("exported validators preserve their public output types", () => { + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); + expectTypeOf< + z.infer + >().toEqualTypeOf(); +}); + +test("evaluation parameters retain custom schema inference", () => { + const parameters = { + subject: z.string(), + model: { type: "model" as const }, + prompt: { type: "prompt" as const }, + } satisfies EvalParameters; + + expectTypeOf>().toEqualTypeOf<{ + subject: string; + model: string; + prompt: Prompt; + }>(); +}); diff --git a/js/src/reporters/types.ts b/js/src/reporters/types.ts index f68724bf6..a85be9bdb 100644 --- a/js/src/reporters/types.ts +++ b/js/src/reporters/types.ts @@ -32,7 +32,7 @@ export interface ReporterBody { /** * A function that takes all evaluator results and returns a boolean indicating - * whether the run was successful. If you return false, the `braintrust eval` + * whether the run was successful. If you return false, the `bt eval` * command will exit with a non-zero status code. * * @param reports diff --git a/js/src/sandbox.ts b/js/src/sandbox.ts index 8517c801d..cfafa8c98 100644 --- a/js/src/sandbox.ts +++ b/js/src/sandbox.ts @@ -1,6 +1,6 @@ import { z } from "zod/v3"; import { slugify } from "../util/string_util"; -import { type IfExistsType } from "./generated_types"; +import type { IfExistsType } from "./generated_plain_types"; import { type BraintrustState, _internalGetGlobalState } from "./logger"; /** diff --git a/js/src/vendor-sdk-types/eve.ts b/js/src/vendor-sdk-types/eve.ts index 1910b3a87..5499fceb9 100644 --- a/js/src/vendor-sdk-types/eve.ts +++ b/js/src/vendor-sdk-types/eve.ts @@ -1,8 +1,8 @@ /** - * Vendored types for eve's authored hook APIs. + * Vendored types for eve's instrumentation provider API, introduced in 0.34.0. * * Keep this surface intentionally narrow. These types are not exported to SDK - * users and should only cover fields we read, correlate, or log. + * users and should only cover fields the Braintrust provider reads or returns. */ export type EveJsonValue = @@ -13,499 +13,197 @@ export type EveJsonValue = | EveJsonValue[] | { readonly [key: string]: EveJsonValue }; -export type EveJsonObject = { readonly [key: string]: EveJsonValue }; - -export interface EveHookContext { - readonly session: { - readonly id: string; - readonly parent?: { - readonly callId?: string; - readonly sessionId?: string; - readonly turn?: { - readonly id?: string; - }; - }; - }; +export interface EveInstrumentationState { + get(): EveJsonValue | undefined; + set(value: EveJsonValue | undefined): void; } -export type EveAssistantStepFinishReason = - | "content-filter" - | "error" - | "length" - | "other" - | "stop" - | "tool-calls"; +export interface EveInstrumentationHandlerContext { + readonly state: EveInstrumentationState; +} -export interface EveStreamEventMeta { - readonly at: string; +export interface EveInstrumentationSetupContext { + readonly agentName: string; + readonly environment?: "development" | "preview" | "production"; + readonly evaluation?: { readonly runId: string }; + readonly frameworkVersion?: string; } -export interface EveRuntimeToolCallActionRequest { - readonly callId: string; - readonly input: EveJsonObject; - readonly kind: "tool-call"; - readonly toolName: string; +export interface EveInstrumentationAttemptScope { + readonly attemptId: string; + readonly attemptIndex: number; + readonly rootSessionId?: string; + readonly sessionId: string; + readonly stepIndex: number; + readonly turnId: string; } -export interface EveRuntimeToolResultActionResult { +export interface EveInstrumentationParentLineage { readonly callId: string; - readonly isError?: boolean; - readonly kind: "tool-result"; - readonly output: EveJsonValue; - readonly toolName: string; + readonly sessionId: string; + readonly turnId: string; } -export type EveRuntimeActionRequest = - | EveRuntimeToolCallActionRequest - | { - readonly callId: string; - readonly input?: EveJsonObject; - readonly kind: "load-skill" | "remote-agent-call"; - readonly name?: string; - } - | { - readonly callId: string; - readonly input: EveJsonObject; - readonly kind: "subagent-call"; - readonly name?: string; - readonly subagentName?: string; - }; - -export type EveRuntimeActionResult = - | EveRuntimeToolResultActionResult - | { - readonly callId: string; - readonly isError?: boolean; - readonly kind: "load-skill-result"; - readonly output?: EveJsonValue; - readonly name?: string; - } - | { - readonly callId: string; - readonly isError?: boolean; - readonly kind: "subagent-result"; - readonly output?: EveJsonValue; - readonly subagentName?: string; - }; +export interface EveInstrumentationTraceContext { + readonly spanId: string; + readonly traceFlags: number; + readonly traceId: string; +} -export type EveActionResultStatus = "completed" | "failed" | "rejected"; +export interface EveInstrumentationUsage { + readonly inputTokenDetails?: { + readonly cacheReadTokens?: number; + readonly cacheWriteTokens?: number; + }; + readonly inputTokens?: number; + readonly outputTokens?: number; +} -export interface EveActionResultError { - readonly code: string; - readonly message: string; +export interface EveInstrumentationModelInput { + readonly instructions?: unknown; + readonly messages: readonly unknown[]; } -export type EveHandleMessageStreamEvent = - | { - readonly data: { - readonly invocation?: unknown; - readonly runtime?: { - readonly agentId: string; - readonly agentName?: string; - readonly eveVersion: string; - readonly modelId: string; - }; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.started"; - } +export type EveInstrumentationContentPart = | { - readonly data: { - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.started"; + readonly text: string; + readonly type: "text"; } | { - readonly data: { - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.completed"; + readonly text: string; + readonly type: "reasoning"; } | { - readonly data: { - readonly message: string; - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "message.received"; - } - | { - readonly data: { - readonly finishReason: EveAssistantStepFinishReason; - readonly message: string | null; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "message.completed"; - } - | { - readonly data: { - readonly reasoning: string; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "reasoning.completed"; - } - | { - readonly data: { - readonly result: EveJsonValue; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "result.completed"; - } - | { - readonly data: { - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.started"; - } - | { - readonly data: { - readonly finishReason: EveAssistantStepFinishReason; - readonly providerMetadata?: { - readonly gateway?: { - readonly generationId?: string; - }; - }; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - readonly usage?: { - readonly cacheReadTokens?: number; - readonly cacheWriteTokens?: number; - readonly costUsd?: number; - readonly inputTokens?: number; - readonly outputTokens?: number; - }; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.completed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "step.failed"; - } - | { - readonly data: { - readonly actions: readonly EveRuntimeActionRequest[]; - readonly sequence: number; - readonly stepIndex: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "actions.requested"; - } - | { - readonly data: { - readonly error?: EveActionResultError; - readonly result: EveRuntimeActionResult; - readonly sequence: number; - readonly stepIndex: number; - readonly status: EveActionResultStatus; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "action.result"; - } - | { - readonly data: { - readonly callId: string; - readonly childSessionId: string; - readonly name: string; - readonly remote?: { - readonly url?: string; - }; - readonly sequence: number; - readonly toolName?: string; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "subagent.called"; - } - | { - readonly data: { - readonly callId: string; - readonly error?: EveActionResultError; - readonly output?: EveJsonValue; - readonly sequence: number; - readonly status?: EveActionResultStatus; - readonly subagentName: string; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "subagent.completed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sequence: number; - readonly turnId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "turn.failed"; - } - | { - readonly data: { - readonly code: string; - readonly details?: EveJsonObject; - readonly message: string; - readonly sessionId: string; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.failed"; + readonly callId: string; + readonly input: unknown; + readonly toolName: string; + readonly type: "tool-call"; } | { - readonly data: { - readonly wait: "next-user-message"; - }; - readonly meta?: EveStreamEventMeta; - readonly type: "session.waiting"; + readonly type: "tool-result"; } | { - readonly meta?: EveStreamEventMeta; - readonly type: "session.completed"; + readonly type: "tool-error"; }; -export interface EveHookDefinition { - readonly events?: { - readonly "*"?: ( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise; - readonly [eventType: string]: - | (( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise) - | undefined; - }; +export interface EveInstrumentationTurnStartedEvent { + readonly idempotencyKey: string; + readonly parentLineage?: EveInstrumentationParentLineage; + readonly parentTraceContext?: EveInstrumentationTraceContext; + readonly rootSessionId: string; + readonly sequence: number; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.started"; } -export interface EveInstrumentationSetupContext { - readonly agentName: string; +export interface EveInstrumentationTurnSettledEvent { + readonly idempotencyKey: string; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.cancelled" | "turn.completed"; } -type EveTextPart = { - readonly text: string; - readonly type: "text"; -}; - -type EveImagePart = { - readonly image: unknown; - readonly mediaType?: string; - readonly type: "image"; -}; - -type EveFilePart = { - readonly data: unknown; - readonly filename?: string; - readonly mediaType: string; - readonly type: "file"; -}; - -type EveReasoningPart = { - readonly text: string; - readonly type: "reasoning"; -}; - -type EveReasoningFilePart = { - readonly data: unknown; - readonly mediaType: string; - readonly type: "reasoning-file"; -}; - -type EveCustomPart = { - readonly kind: `${string}.${string}`; - readonly type: "custom"; -}; - -type EveToolCallPart = { - readonly input: unknown; - readonly providerExecuted?: boolean; - readonly toolCallId: string; - readonly toolName: string; - readonly type: "tool-call"; -}; - -type EveToolResultContentPart = - | EveTextPart - | EveFilePart - | { - readonly data: string; - readonly filename?: string; - readonly mediaType: string; - readonly type: "file-data"; - } - | { - readonly mediaType?: string; - readonly type: "file-url"; - readonly url: string; - } - | { - readonly fileId: string | Readonly>; - readonly type: "file-id" | "image-file-id"; - } - | { - readonly providerReference: Readonly>; - readonly type: "file-reference" | "image-file-reference"; - } - | { - readonly data: string; - readonly mediaType: string; - readonly type: "image-data"; - } - | { - readonly type: "image-url"; - readonly url: string; - } - | { readonly type: "custom" }; - -type EveToolResultOutput = - | { - readonly type: "text" | "error-text"; - readonly value: string; - } - | { - readonly type: "json" | "error-json"; - readonly value: EveJsonValue; - } - | { - readonly reason?: string; - readonly type: "execution-denied"; - } - | { - readonly type: "content"; - readonly value: readonly EveToolResultContentPart[]; - }; +export interface EveInstrumentationTurnFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly sessionId: string; + readonly turnId: string; + readonly type: "turn.failed"; +} -type EveToolResultPart = { - readonly output: EveToolResultOutput; - readonly toolCallId: string; - readonly toolName: string; - readonly type: "tool-result"; -}; +export interface EveInstrumentationModelCallStartedEvent { + readonly idempotencyKey: string; + readonly input?: EveInstrumentationModelInput; + readonly model: { + readonly modelId: string; + readonly provider: string; + }; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.started"; +} -type EveToolApprovalRequest = { - readonly approvalId: string; - readonly isAutomatic?: boolean; - readonly signature?: string; - readonly toolCallId: string; - readonly type: "tool-approval-request"; -}; +export interface EveInstrumentationModelCallCompletedEvent { + readonly content?: readonly EveInstrumentationContentPart[]; + readonly finishReason: string; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.completed"; + readonly usage: EveInstrumentationUsage; +} -type EveToolApprovalResponse = { - readonly approvalId: string; - readonly approved: boolean; - readonly providerExecuted?: boolean; - readonly reason?: string; - readonly type: "tool-approval-response"; -}; +export interface EveInstrumentationModelCallFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "model.call.failed"; +} -export type EveModelMessageContentPart = - | EveTextPart - | EveImagePart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveCustomPart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - | EveToolApprovalResponse - | EveToolResultContentPart; +export interface EveInstrumentationStepAttemptCompletedEvent { + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "step.attempt.completed"; +} -export type EveSystemModelMessage = { - readonly content: string; - readonly role: "system"; -}; +export interface EveInstrumentationStepAttemptFailedEvent { + readonly error?: unknown; + readonly idempotencyKey: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "step.attempt.failed"; +} -export type EveModelMessage = - | EveSystemModelMessage - | { - readonly content: - | string - | readonly (EveTextPart | EveImagePart | EveFilePart)[]; - readonly role: "user"; - } - | { - readonly content: - | string - | readonly ( - | EveTextPart - | EveCustomPart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - )[]; - readonly role: "assistant"; - } - | { - readonly content: readonly ( - | EveToolResultPart - | EveToolApprovalResponse - )[]; - readonly role: "tool"; - }; +export interface EveInstrumentationActionStartedEvent { + readonly callId: string; + readonly idempotencyKey: string; + readonly input?: unknown; + readonly name: string; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.started"; +} -export interface EveInstrumentationModelInput { - readonly instructions?: string | readonly EveSystemModelMessage[]; - readonly messages: readonly EveModelMessage[]; +export interface EveInstrumentationActionCompletedEvent { + readonly acceptedAtMs?: number; + readonly idempotencyKey: string; + readonly output: + | { readonly output?: unknown; readonly type: "result" } + | { readonly error?: unknown; readonly type: "error" }; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.completed"; } -export interface EveInstrumentationStepStartedEventInput { - readonly modelInput: EveInstrumentationModelInput; - readonly session: { - readonly id: string; - }; - readonly step: { - readonly index: number; - }; - readonly turn: { - readonly id: string; - readonly sequence: number; - }; +export interface EveInstrumentationActionFailedEvent { + readonly acceptedAtMs?: number; + readonly error?: unknown; + readonly errorCode?: string; + readonly idempotencyKey: string; + readonly outcome: "abandoned" | "cancelled" | "failed" | "rejected"; + readonly scope: EveInstrumentationAttemptScope; + readonly type: "action.failed"; } +type EveInstrumentationHandler = ( + event: TEvent, + context: EveInstrumentationHandlerContext, +) => void | PromiseLike; + export interface EveInstrumentationDefinition { - readonly events?: { - readonly "step.started"?: ( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }; - readonly [eventType: string]: - | (( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }) - | undefined; + readonly capture: "content"; + readonly events: { + readonly "action.completed": EveInstrumentationHandler; + readonly "action.failed": EveInstrumentationHandler; + readonly "action.started": EveInstrumentationHandler; + readonly "model.call.completed": EveInstrumentationHandler; + readonly "model.call.failed": EveInstrumentationHandler; + readonly "model.call.started": EveInstrumentationHandler; + readonly "step.attempt.completed": EveInstrumentationHandler; + readonly "step.attempt.failed": EveInstrumentationHandler; + readonly "turn.cancelled": EveInstrumentationHandler; + readonly "turn.completed": EveInstrumentationHandler; + readonly "turn.failed": EveInstrumentationHandler; + readonly "turn.started": EveInstrumentationHandler; }; - readonly recordInputs?: boolean; - readonly recordOutputs?: boolean; - readonly setup?: (context: EveInstrumentationSetupContext) => void; + readonly flush: () => Promise; + readonly setup?: ( + context: EveInstrumentationSetupContext, + ) => void | PromiseLike; } diff --git a/js/src/vendor-sdk-types/flue.ts b/js/src/vendor-sdk-types/flue.ts index 367046fe2..2f7c75222 100644 --- a/js/src/vendor-sdk-types/flue.ts +++ b/js/src/vendor-sdk-types/flue.ts @@ -1,5 +1,5 @@ /** - * Vendored types for @flue/runtime observe/instrument-based instrumentation. + * Vendored types for @flue/runtime instrumentation. * * Keep this surface intentionally narrow. These types are not exported to SDK * users and should only cover fields we read, correlate, or log. @@ -16,17 +16,11 @@ export interface FlueUsage { cacheWrite?: number; totalTokens?: number; cost?: { - input?: number; - output?: number; - cacheRead?: number; - cacheWrite?: number; total?: number; }; } export interface FlueBaseEvent { - type?: string; - v?: number; runId?: string; instanceId?: string; submissionId?: string; @@ -41,7 +35,6 @@ export interface FlueBaseEvent { harness?: string; operationId?: string; turnId?: string; - [key: string]: unknown; } export interface FlueRunStartEvent extends FlueBaseEvent { @@ -49,13 +42,7 @@ export interface FlueRunStartEvent extends FlueBaseEvent { runId: string; startedAt?: string; workflowName?: string; - owner?: { - kind?: string; - workflowName?: string; - instanceId?: string; - }; input?: unknown; - payload?: unknown; } export interface FlueRunResumeEvent extends FlueBaseEvent { @@ -91,10 +78,9 @@ export interface FlueOperationEvent extends FlueBaseEvent { result?: unknown; agentInput?: { text?: string; - images?: Array<{ mimeType?: string }>; }; agentOutput?: - | { type?: "text"; text?: string; finishReason?: string } + | { type?: "text"; text?: string } | { type?: "data"; data?: unknown }; usage?: FlueUsage; } @@ -103,14 +89,6 @@ export interface FlueTurnRequestEvent extends FlueBaseEvent { type: "turn_request"; turnId: string; purpose: FlueTurnPurpose; - model?: string; - provider?: string; - api?: string; - input?: { - systemPrompt?: string; - messages?: unknown[]; - tools?: unknown[]; - }; request?: { api?: string; input?: { @@ -118,14 +96,11 @@ export interface FlueTurnRequestEvent extends FlueBaseEvent { messages?: unknown[]; tools?: unknown[]; }; - model?: string; requestedModel?: string; providerId?: string; providerName?: string; - reasoning?: string; reasoningLevel?: string; }; - reasoning?: string; } export interface FlueTurnEvent extends FlueBaseEvent { @@ -133,12 +108,8 @@ export interface FlueTurnEvent extends FlueBaseEvent { turnId: string; purpose?: FlueTurnPurpose; durationMs?: number; - model?: string; - provider?: string; - api?: string; request?: { api?: string; - model?: string; requestedModel?: string; providerId?: string; providerName?: string; @@ -146,17 +117,11 @@ export interface FlueTurnEvent extends FlueBaseEvent { response?: { output?: unknown; usage?: FlueUsage; - stopReason?: string; finishReason?: string; responseModel?: string; error?: unknown; - errorInfo?: { type?: string; message?: string }; }; - output?: unknown; - usage?: FlueUsage; - stopReason?: string; isError?: boolean; - error?: unknown; } export interface FlueToolStartEvent extends FlueBaseEvent { @@ -164,21 +129,15 @@ export interface FlueToolStartEvent extends FlueBaseEvent { toolName?: string; toolCallId: string; args?: unknown; - arguments?: unknown; - input?: unknown; - origin?: "model" | "caller" | "framework" | "adapter"; - description?: string; } export interface FlueToolCallEvent extends FlueBaseEvent { - type: "tool_call" | "tool"; + type: "tool"; toolName?: string; toolCallId: string; isError?: boolean; result?: unknown; effectiveResult?: unknown; - output?: unknown; - error?: unknown; errorInfo?: { type?: string; message?: string }; durationMs?: number; } @@ -197,9 +156,6 @@ export interface FlueTaskEvent extends FlueBaseEvent { agent?: string; isError?: boolean; result?: unknown; - agentOutput?: - | { type?: "text"; text?: string; finishReason?: string } - | { type?: "data"; data?: unknown }; errorInfo?: { type?: string; message?: string }; durationMs?: number; } @@ -222,12 +178,7 @@ export interface FlueCompactionEvent extends FlueBaseEvent { } export interface FlueContext { - readonly id?: string; - readonly runId?: string; -} - -export interface FlueObservableContext extends FlueContext { - subscribeEvent(callback: (event: unknown) => unknown): () => void; + readonly id: string; } export type FlueExecutionOperation = @@ -244,25 +195,12 @@ export type FlueExecutionOperation = | { type: "task"; taskId: string } | { type: "coordinator"; phase: "reconcile" }; -export interface FlueTraceCarrier { - traceparent: string; - tracestate?: string; -} - export interface FlueExecutionContext { eventContext?: FlueContext; runId?: string; - instanceId?: string; - submissionId?: string; - dispatchId?: string; - agentName?: string; - conversationId?: string; - harness?: string; - session?: string; operationId?: string; turnId?: string; taskId?: string; - traceCarrier?: FlueTraceCarrier; } export type FlueExecutionInterceptor = ( diff --git a/js/src/wrappers/ai-sdk/ai-sdk.ts b/js/src/wrappers/ai-sdk/ai-sdk.ts index 63de58ea6..dc6c7b09d 100644 --- a/js/src/wrappers/ai-sdk/ai-sdk.ts +++ b/js/src/wrappers/ai-sdk/ai-sdk.ts @@ -4,7 +4,7 @@ import { SpanTypeAttribute } from "../../../util"; import { aiSDKChannels, harnessAgentChannels, -} from "../../instrumentation/plugins/ai-sdk-channels"; +} from "../../instrumentation/providers/ai-sdk-channels"; import type { AISDK, AISDKAgentClass, diff --git a/js/src/wrappers/ai-sdk/harness-agent-context.ts b/js/src/wrappers/ai-sdk/harness-agent-context.ts index fc0b6171a..f17f2a442 100644 --- a/js/src/wrappers/ai-sdk/harness-agent-context.ts +++ b/js/src/wrappers/ai-sdk/harness-agent-context.ts @@ -335,10 +335,10 @@ export function currentHarnessTurnParent(): HarnessTurnParent | undefined { export function bindHarnessTurnParentToStart( tracingChannel: IsoTracingChannel, parentFromEvent: (event: T) => HarnessTurnParent | undefined, -): () => void { +): void { const startChannel = tracingChannel.start; if (!startChannel) { - return () => {}; + return; } harnessTurnParentStore ??= iso.newAsyncLocalStorage< @@ -349,9 +349,6 @@ export function bindHarnessTurnParentToStart( store, (event) => parentFromEvent(event) ?? store.getStore(), ); - return () => { - startChannel.unbindStore(store); - }; } export function startHarnessTurnChildSpan( diff --git a/js/src/wrappers/ai-sdk/telemetry.ts b/js/src/wrappers/ai-sdk/telemetry.ts index 1fb2c9812..110f8e221 100644 --- a/js/src/wrappers/ai-sdk/telemetry.ts +++ b/js/src/wrappers/ai-sdk/telemetry.ts @@ -20,7 +20,7 @@ import { processAISDKWorkflowAgentCallInput, processAISDKWorkflowAgentModelCallInput, serializeModelWithProvider, -} from "../../instrumentation/plugins/ai-sdk-plugin"; +} from "../../instrumentation/providers/ai-sdk-instrumentation"; import type { AISDKCallParams, AISDKEmbeddingResult, diff --git a/js/src/wrappers/anthropic.ts b/js/src/wrappers/anthropic.ts index 12aae0218..e4cfc708f 100644 --- a/js/src/wrappers/anthropic.ts +++ b/js/src/wrappers/anthropic.ts @@ -1,4 +1,4 @@ -import { anthropicChannels } from "../instrumentation/plugins/anthropic-channels"; +import { anthropicChannels } from "../instrumentation/providers/anthropic-channels"; import { TypedApplyProxy } from "../typed-instrumentation-helpers"; import type { AnthropicBeta, diff --git a/js/src/wrappers/attachment-utils.ts b/js/src/wrappers/attachment-utils.ts index 7fb2b96f5..df7544fce 100644 --- a/js/src/wrappers/attachment-utils.ts +++ b/js/src/wrappers/attachment-utils.ts @@ -242,6 +242,26 @@ export function processInputAttachments(input: any): any { } } + // AI SDK tool output content format + if ( + (node.type === "image-data" || node.type === "file-data") && + node.data + ) { + const mediaType = node.mediaType || "application/octet-stream"; + const filename = + node.filename || + `output_${node.type === "image-data" ? "image" : "file"}_${attachmentIndex}.${getExtensionFromMediaType(mediaType)}`; + const attachment = toAttachment(node.data, mediaType, filename); + + if (attachment) { + attachmentIndex++; + return { + ...node, + data: attachment, + }; + } + } + const processed: Record = {}; for (const [key, value] of Object.entries(node)) { processed[key] = processNode(value); diff --git a/js/src/wrappers/bedrock-runtime.ts b/js/src/wrappers/bedrock-runtime.ts index 13fcd1844..4e60fd912 100644 --- a/js/src/wrappers/bedrock-runtime.ts +++ b/js/src/wrappers/bedrock-runtime.ts @@ -1,9 +1,9 @@ import { runWithAutoInstrumentationSuppressed } from "../instrumentation/auto-instrumentation-suppression"; -import { bedrockRuntimeChannels } from "../instrumentation/plugins/bedrock-runtime-channels"; +import { bedrockRuntimeChannels } from "../instrumentation/providers/bedrock-runtime-channels"; import { buildBedrockRuntimeSpanInfo, getBedrockRuntimeOperation, -} from "../instrumentation/plugins/bedrock-runtime-common"; +} from "../instrumentation/providers/bedrock-runtime-common"; import type { BedrockRuntimeClient, BedrockRuntimeCommandLike, diff --git a/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts b/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts index db24f0bcf..fae63e64d 100644 --- a/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts +++ b/js/src/wrappers/claude-agent-sdk/claude-agent-sdk.ts @@ -1,6 +1,6 @@ -import { claudeAgentSDKChannels } from "../../instrumentation/plugins/claude-agent-sdk-channels"; -import { CLAUDE_AGENT_SDK_SKIP_LOCAL_TOOL_HOOKS_OPTION } from "../../instrumentation/plugins/claude-agent-sdk-instrumentation-constants"; -import { wrapLocalClaudeToolHandler } from "../../instrumentation/plugins/claude-agent-sdk-local-tool-spans"; +import { claudeAgentSDKChannels } from "../../instrumentation/providers/claude-agent-sdk-channels"; +import { CLAUDE_AGENT_SDK_SKIP_LOCAL_TOOL_HOOKS_OPTION } from "../../instrumentation/providers/claude-agent-sdk-instrumentation-constants"; +import { wrapLocalClaudeToolHandler } from "../../instrumentation/providers/claude-agent-sdk-local-tool-spans"; import type { ClaudeAgentSDKModule, ClaudeAgentSDKQueryParams, diff --git a/js/src/wrappers/cloudflare-agent.ts b/js/src/wrappers/cloudflare-agent.ts index c34bb6134..baee1d63d 100644 --- a/js/src/wrappers/cloudflare-agent.ts +++ b/js/src/wrappers/cloudflare-agent.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { cloudflareAgentsChannels } from "../instrumentation/plugins/cloudflare-agents-channels"; +import { cloudflareAgentsChannels } from "../instrumentation/providers/cloudflare-agents-channels"; import type { CloudflareAgent } from "../vendor-sdk-types/cloudflare-agents"; const WRAPPED_RUN_AGENT_TOOL = Symbol.for( diff --git a/js/src/wrappers/cloudflare-ai-chat.ts b/js/src/wrappers/cloudflare-ai-chat.ts index 87df27775..11f834feb 100644 --- a/js/src/wrappers/cloudflare-ai-chat.ts +++ b/js/src/wrappers/cloudflare-ai-chat.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { instrumentCloudflareAIChatAgent } from "../instrumentation/plugins/cloudflare-ai-chat-instrumentation"; +import { instrumentCloudflareAIChatAgent } from "../instrumentation/providers/cloudflare-ai-chat-instrumentation"; import type { CloudflareAIChatAgent, CloudflareAIChatAgentConstructor, diff --git a/js/src/wrappers/cloudflare-think.ts b/js/src/wrappers/cloudflare-think.ts index 9c04400ce..2b1e371bd 100644 --- a/js/src/wrappers/cloudflare-think.ts +++ b/js/src/wrappers/cloudflare-think.ts @@ -1,4 +1,4 @@ -import { cloudflareThinkChannels } from "../instrumentation/plugins/cloudflare-think-channels"; +import { cloudflareThinkChannels } from "../instrumentation/providers/cloudflare-think-channels"; import type { CloudflareThinkConstructor, CloudflareThinkInstance, diff --git a/js/src/wrappers/cohere.ts b/js/src/wrappers/cohere.ts index aa6038030..6c55f65dd 100644 --- a/js/src/wrappers/cohere.ts +++ b/js/src/wrappers/cohere.ts @@ -1,4 +1,4 @@ -import { cohereChannels } from "../instrumentation/plugins/cohere-channels"; +import { cohereChannels } from "../instrumentation/providers/cohere-channels"; import type { CohereChatRequest, CohereChatResponse, diff --git a/js/src/wrappers/cursor-sdk.ts b/js/src/wrappers/cursor-sdk.ts index e20f41fba..c20897b82 100644 --- a/js/src/wrappers/cursor-sdk.ts +++ b/js/src/wrappers/cursor-sdk.ts @@ -1,4 +1,4 @@ -import { cursorSDKChannels } from "../instrumentation/plugins/cursor-sdk-channels"; +import { cursorSDKChannels } from "../instrumentation/providers/cursor-sdk-channels"; import type { CursorSDKAgent, CursorSDKAgentClass, diff --git a/js/src/wrappers/genkit.test.ts b/js/src/wrappers/genkit.test.ts index 5363a64eb..3677dc99d 100644 --- a/js/src/wrappers/genkit.test.ts +++ b/js/src/wrappers/genkit.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { IsoChannelHandlers } from "../isomorph"; import type { ChannelMessage } from "../instrumentation/core/channel-definitions"; -import { genkitChannels } from "../instrumentation/plugins/genkit-channels"; +import { genkitChannels } from "../instrumentation/providers/genkit-channels"; import { configureNode } from "../node/config"; import type { GenkitAction, diff --git a/js/src/wrappers/genkit.ts b/js/src/wrappers/genkit.ts index fc35a8426..a4900ae9f 100644 --- a/js/src/wrappers/genkit.ts +++ b/js/src/wrappers/genkit.ts @@ -1,4 +1,4 @@ -import { genkitChannels } from "../instrumentation/plugins/genkit-channels"; +import { genkitChannels } from "../instrumentation/providers/genkit-channels"; import type { GenkitAction, GenkitEmbedManyParams, diff --git a/js/src/wrappers/github-copilot.ts b/js/src/wrappers/github-copilot.ts index 35e68fa87..20e76dbda 100644 --- a/js/src/wrappers/github-copilot.ts +++ b/js/src/wrappers/github-copilot.ts @@ -1,4 +1,4 @@ -import { gitHubCopilotChannels } from "../instrumentation/plugins/github-copilot-channels"; +import { gitHubCopilotChannels } from "../instrumentation/providers/github-copilot-channels"; import type { GitHubCopilotClient, GitHubCopilotResumeSessionConfig, diff --git a/js/src/wrappers/google-adk.ts b/js/src/wrappers/google-adk.ts index 28af7395e..6e908222e 100644 --- a/js/src/wrappers/google-adk.ts +++ b/js/src/wrappers/google-adk.ts @@ -1,4 +1,4 @@ -import { googleADKChannels } from "../instrumentation/plugins/google-adk-channels"; +import { googleADKChannels } from "../instrumentation/providers/google-adk-channels"; import type { GoogleADKRunner, GoogleADKRunnerConstructor, diff --git a/js/src/wrappers/google-genai.ts b/js/src/wrappers/google-genai.ts index 192f79c1a..7d21e8eea 100644 --- a/js/src/wrappers/google-genai.ts +++ b/js/src/wrappers/google-genai.ts @@ -1,4 +1,4 @@ -import { googleGenAIChannels } from "../instrumentation/plugins/google-genai-channels"; +import { googleGenAIChannels } from "../instrumentation/providers/google-genai-channels"; import { isObject } from "../util"; import type { GoogleGenAIClient, diff --git a/js/src/wrappers/groq.ts b/js/src/wrappers/groq.ts index 68312fa81..c736e00c8 100644 --- a/js/src/wrappers/groq.ts +++ b/js/src/wrappers/groq.ts @@ -1,4 +1,4 @@ -import { groqChannels } from "../instrumentation/plugins/groq-channels"; +import { groqChannels } from "../instrumentation/providers/groq-channels"; import type { GroqChat, GroqChatCompletion, diff --git a/js/src/wrappers/huggingface-transformers.ts b/js/src/wrappers/huggingface-transformers.ts index 3965f1234..52d8e2d56 100644 --- a/js/src/wrappers/huggingface-transformers.ts +++ b/js/src/wrappers/huggingface-transformers.ts @@ -1,7 +1,7 @@ import { huggingFaceTransformersChannels, isSupportedHuggingFaceTransformersTask, -} from "../instrumentation/plugins/huggingface-transformers-channels"; +} from "../instrumentation/providers/huggingface-transformers-channels"; import type { HuggingFaceTransformersModule, HuggingFaceTransformersPipeline, diff --git a/js/src/wrappers/huggingface.ts b/js/src/wrappers/huggingface.ts index 6ff98a259..13a58f515 100644 --- a/js/src/wrappers/huggingface.ts +++ b/js/src/wrappers/huggingface.ts @@ -1,4 +1,4 @@ -import { huggingFaceChannels } from "../instrumentation/plugins/huggingface-channels"; +import { huggingFaceChannels } from "../instrumentation/providers/huggingface-channels"; import { isObject } from "../../util"; import type { HuggingFaceChatCompletion, diff --git a/js/src/wrappers/langsmith.ts b/js/src/wrappers/langsmith.ts index a8486c573..983f42dd8 100644 --- a/js/src/wrappers/langsmith.ts +++ b/js/src/wrappers/langsmith.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { langSmithChannels } from "../instrumentation/plugins/langsmith-channels"; +import { langSmithChannels } from "../instrumentation/providers/langsmith-channels"; import type { LangSmithClient, LangSmithClientConstructor, diff --git a/js/src/wrappers/mistral.ts b/js/src/wrappers/mistral.ts index c449997f7..4771dda38 100644 --- a/js/src/wrappers/mistral.ts +++ b/js/src/wrappers/mistral.ts @@ -1,4 +1,4 @@ -import { mistralChannels } from "../instrumentation/plugins/mistral-channels"; +import { mistralChannels } from "../instrumentation/providers/mistral-channels"; import type { MistralAgents, MistralAgentsCompletionResponse, diff --git a/js/src/wrappers/oai.ts b/js/src/wrappers/oai.ts index b2fa0850e..743460b7e 100644 --- a/js/src/wrappers/oai.ts +++ b/js/src/wrappers/oai.ts @@ -10,7 +10,7 @@ import type { ArgsOf, ResultOf, } from "../instrumentation/core/channel-definitions"; -import { openAIChannels } from "../instrumentation/plugins/openai-channels"; +import { openAIChannels } from "../instrumentation/providers/openai-channels"; import type { OpenAIChatCompletion, OpenAIChatCreateParams, diff --git a/js/src/wrappers/oai_responses.ts b/js/src/wrappers/oai_responses.ts index 8e936ebbc..432b1281a 100644 --- a/js/src/wrappers/oai_responses.ts +++ b/js/src/wrappers/oai_responses.ts @@ -3,7 +3,7 @@ import type { ResultOf, } from "../instrumentation/core/channel-definitions"; import type { ChannelSpanInfo } from "../instrumentation/core/types"; -import { openAIChannels } from "../instrumentation/plugins/openai-channels"; +import { openAIChannels } from "../instrumentation/providers/openai-channels"; import { parseMetricsFromUsage } from "../openai-utils"; import { APIPromise, diff --git a/js/src/wrappers/ollama.test.ts b/js/src/wrappers/ollama.test.ts index 9a8ebff46..b07dac5f4 100644 --- a/js/src/wrappers/ollama.test.ts +++ b/js/src/wrappers/ollama.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest"; import { debugLogger } from "../debug-logger"; -import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import { ollamaChannels } from "../instrumentation/providers/ollama-channels"; import type { OllamaClient } from "../vendor-sdk-types/ollama"; import { wrapOllama } from "./ollama"; diff --git a/js/src/wrappers/ollama.ts b/js/src/wrappers/ollama.ts index 3f90067a9..bcc48a937 100644 --- a/js/src/wrappers/ollama.ts +++ b/js/src/wrappers/ollama.ts @@ -1,5 +1,5 @@ import { debugLogger } from "../debug-logger"; -import { ollamaChannels } from "../instrumentation/plugins/ollama-channels"; +import { ollamaChannels } from "../instrumentation/providers/ollama-channels"; import { isObject } from "../../util"; import type { OllamaChatRequest, diff --git a/js/src/wrappers/openai-codex.ts b/js/src/wrappers/openai-codex.ts index 8b5a435b3..c4225e9c0 100644 --- a/js/src/wrappers/openai-codex.ts +++ b/js/src/wrappers/openai-codex.ts @@ -1,4 +1,4 @@ -import { openAICodexChannels } from "../instrumentation/plugins/openai-codex-channels"; +import { openAICodexChannels } from "../instrumentation/providers/openai-codex-channels"; import type { OpenAICodexClass, OpenAICodexClient, diff --git a/js/src/wrappers/openai-promise-utils.ts b/js/src/wrappers/openai-promise-utils.ts index afa23586c..b41e57f4b 100644 --- a/js/src/wrappers/openai-promise-utils.ts +++ b/js/src/wrappers/openai-promise-utils.ts @@ -6,7 +6,7 @@ import type { OpenAIAsyncChannel, OpenAIChannel, OpenAIStartContext, -} from "../instrumentation/plugins/openai-channels"; +} from "../instrumentation/providers/openai-channels"; export type EnhancedResponse = { response: Response; diff --git a/js/src/wrappers/openrouter-agent.test.ts b/js/src/wrappers/openrouter-agent.test.ts index 02f01e040..c1c94e12f 100644 --- a/js/src/wrappers/openrouter-agent.test.ts +++ b/js/src/wrappers/openrouter-agent.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { wrapOpenRouterAgent } from "./openrouter-agent"; -import { openRouterAgentChannels } from "../instrumentation/plugins/openrouter-agent-channels"; +import { openRouterAgentChannels } from "../instrumentation/providers/openrouter-agent-channels"; describe("wrapOpenRouterAgent", () => { afterEach(() => { diff --git a/js/src/wrappers/openrouter-agent.ts b/js/src/wrappers/openrouter-agent.ts index 07328249b..336b2d84f 100644 --- a/js/src/wrappers/openrouter-agent.ts +++ b/js/src/wrappers/openrouter-agent.ts @@ -1,4 +1,4 @@ -import { openRouterAgentChannels } from "../instrumentation/plugins/openrouter-agent-channels"; +import { openRouterAgentChannels } from "../instrumentation/providers/openrouter-agent-channels"; import type { OpenRouterAgentClient, OpenRouterAgentCallModelRequest, diff --git a/js/src/wrappers/openrouter.ts b/js/src/wrappers/openrouter.ts index bfd39d2c1..ab7231b0c 100644 --- a/js/src/wrappers/openrouter.ts +++ b/js/src/wrappers/openrouter.ts @@ -1,4 +1,4 @@ -import { openRouterChannels } from "../instrumentation/plugins/openrouter-channels"; +import { openRouterChannels } from "../instrumentation/providers/openrouter-channels"; import type { OpenRouterBeta, OpenRouterCallModelRequest, diff --git a/js/src/wrappers/pi-coding-agent.ts b/js/src/wrappers/pi-coding-agent.ts index 60565c596..f01f6a75a 100644 --- a/js/src/wrappers/pi-coding-agent.ts +++ b/js/src/wrappers/pi-coding-agent.ts @@ -1,4 +1,4 @@ -import { piCodingAgentChannels } from "../instrumentation/plugins/pi-coding-agent-channels"; +import { piCodingAgentChannels } from "../instrumentation/providers/pi-coding-agent-channels"; import type { PiAgentSession, PiAgentSessionClass, diff --git a/js/src/wrappers/strands-agent-sdk.ts b/js/src/wrappers/strands-agent-sdk.ts index 6c6173034..3b975c75c 100644 --- a/js/src/wrappers/strands-agent-sdk.ts +++ b/js/src/wrappers/strands-agent-sdk.ts @@ -1,4 +1,4 @@ -import { strandsAgentSDKChannels } from "../instrumentation/plugins/strands-agent-sdk-channels"; +import { strandsAgentSDKChannels } from "../instrumentation/providers/strands-agent-sdk-channels"; import type { StrandsAgent, StrandsAgentConstructor, diff --git a/js/src/wrappers/voyageai.ts b/js/src/wrappers/voyageai.ts index 6abc4bb1a..8285282f5 100644 --- a/js/src/wrappers/voyageai.ts +++ b/js/src/wrappers/voyageai.ts @@ -1,4 +1,4 @@ -import { voyageAIChannels } from "../instrumentation/plugins/voyageai-channels"; +import { voyageAIChannels } from "../instrumentation/providers/voyageai-channels"; import type { VoyageAIClient, VoyageAIContextualizedEmbedRequest, diff --git a/js/src/zod/zod-serialization-test-shared.ts b/js/src/zod/zod-serialization-test-shared.ts deleted file mode 100644 index a78f4b708..000000000 --- a/js/src/zod/zod-serialization-test-shared.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Shared test expectations for zod serialization tests - * - * This module contains expected JSON Schema outputs used by both v3 and v4 - * serialization tests to ensure consistent behavior across versions. - */ - -/** - * Expected JSON schema for string parameter with description and default - */ - -export const EXPECTED_STRING_SCHEMA = { - type: "string", - description: "The instructions for the agent", - default: "You are a helpful assistant.", - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for number parameter with constraints - */ - -export const EXPECTED_NUMBER_SCHEMA = { - type: "number", - minimum: 0, - maximum: 2, - description: "Temperature for LLM", - default: 0.7, - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for object parameter - */ - -export const EXPECTED_OBJECT_SCHEMA = { - type: "object", - properties: { - model: { type: "string" }, - maxTokens: { type: "number" }, - }, - additionalProperties: false, - required: ["model"], - description: "Configuration object", - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for enum parameter - */ - -export const EXPECTED_ENUM_SCHEMA = { - type: "string", - enum: ["fast", "accurate", "balanced"], - description: "Processing mode", - default: "balanced", - $schema: "http://json-schema.org/draft-07/schema#", -}; - -/** - * Expected JSON schema for array parameter - */ - -export const EXPECTED_ARRAY_SCHEMA = { - type: "array", - items: { type: "string" }, - description: "Tags for filtering", - default: ["default"], - $schema: "http://json-schema.org/draft-07/schema#", -}; diff --git a/js/src/zod/zod-v3-serialization.test.ts b/js/src/zod/zod-v3-serialization.test.ts deleted file mode 100644 index c4d1097d2..000000000 --- a/js/src/zod/zod-v3-serialization.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Tests for zod v3 compatibility with serializeEvalParametersToStaticParametersSchema - * - * This file tests serializeEvalParametersToStaticParametersSchema specifically with zod v3. - * See zod-v4-serialization.test.ts for zod v4 specific tests. - */ - -import { test, describe, beforeEach, expect } from "vitest"; -import * as zodModule from "zod"; -import { z } from "zod"; -import { - EXPECTED_STRING_SCHEMA, - EXPECTED_NUMBER_SCHEMA, - EXPECTED_OBJECT_SCHEMA, - EXPECTED_ENUM_SCHEMA, - EXPECTED_ARRAY_SCHEMA, -} from "./zod-serialization-test-shared"; -import { - serializeEvalParametersToStaticParametersSchema, - serializeRemoteEvalParametersContainer, -} from "../framework2"; -import { - serializedParametersContainerSchema, - evaluatorDefinitionSchema, -} from "../../dev/types"; -// Detect which zod version is installed by checking for v4-specific properties -function getInstalledZodVersion(): 3 | 4 { - const testSchema = zodModule.z.string(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return "_zod" in (testSchema as any) ? 4 : 3; -} - -describe("serializeEvalParametersToStaticParametersSchema with Zod v3", () => { - beforeEach(() => { - const version = getInstalledZodVersion(); - expect(version).toBe(3); - }); - - test("string schema serializes correctly", () => { - const parameters = { - instructions: z - .string() - .describe("The instructions for the agent") - .default("You are a helpful assistant."), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - console.log(result); - expect(result.instructions).toBeDefined(); - expect(result.instructions.type).toBe("data"); - expect( - result.instructions.type === "data" && result.instructions.schema, - ).toStrictEqual(EXPECTED_STRING_SCHEMA); - expect(result.instructions.description).toBe( - "The instructions for the agent", - ); - expect(result.instructions.default).toBe("You are a helpful assistant."); - }); - - test("number schema serializes correctly", () => { - const parameters = { - temperature: z - .number() - .min(0) - .max(2) - .describe("Temperature for LLM") - .default(0.7), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - result.temperature.type; - - expect(result.temperature.type).toBe("data"); - expect( - result.temperature.type === "data" && result.temperature.schema, - ).toStrictEqual(EXPECTED_NUMBER_SCHEMA); - expect(result.temperature.description).toBe("Temperature for LLM"); - expect(result.temperature.default).toBe(0.7); - }); - - test("object schema serializes correctly", () => { - const parameters = { - config: z - .object({ - model: z.string(), - maxTokens: z.number().optional(), - }) - .describe("Configuration object"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.config.type).toBe("data"); - expect(result.config.type === "data" && result.config.schema).toStrictEqual( - EXPECTED_OBJECT_SCHEMA, - ); - expect(result.config.description).toBe("Configuration object"); - }); - - test("enum schema serializes correctly", () => { - const parameters = { - mode: z - .enum(["fast", "accurate", "balanced"]) - .describe("Processing mode") - .default("balanced"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.mode.type).toBe("data"); - expect(result.mode.type === "data" && result.mode.schema).toStrictEqual( - EXPECTED_ENUM_SCHEMA, - ); - expect(result.mode.description).toBe("Processing mode"); - expect(result.mode.default).toBe("balanced"); - }); - - test("array schema serializes correctly", () => { - const parameters = { - tags: z - .array(z.string()) - .describe("Tags for filtering") - .default(["default"]), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.tags.type).toBe("data"); - expect(result.tags.type === "data" && result.tags.schema).toStrictEqual( - EXPECTED_ARRAY_SCHEMA, - ); - expect(result.tags.description).toBe("Tags for filtering"); - expect(result.tags.default).toEqual(["default"]); - }); - - test("model parameter serializes correctly", () => { - const parameters = { - model: { - type: "model" as const, - description: "Model to use", - default: "gpt-5-mini", - }, - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.model.type).toBe("model"); - expect(result.model.description).toBe("Model to use"); - expect(result.model.default).toBe("gpt-5-mini"); - }); -}); - -describe("serializeRemoteEvalParametersContainer with Zod v3", () => { - beforeEach(() => { - const version = getInstalledZodVersion(); - expect(version).toBe(3); - }); - - test("local parameters are serialized with legacyParameters type", () => { - const parameters = { - instructions: z - .string() - .describe("The instructions for the agent") - .default("You are a helpful assistant."), - }; - - const result = serializeRemoteEvalParametersContainer(parameters); - - expect(result.type).toBe("braintrust.staticParameters"); - expect(result.source).toBeNull(); - if (result.type === "braintrust.staticParameters") { - expect(result.schema.instructions).toBeDefined(); - expect(result.schema.instructions.type).toBe("data"); - } - }); -}); - -describe("serializedParametersContainerSchema parsing", () => { - const schema = { - model: { - type: "model" as const, - description: "Model to use", - default: "gpt-5-mini", - }, - }; - - test("parses braintrust.staticParameters with source: null", () => { - const result = serializedParametersContainerSchema.safeParse({ - type: "braintrust.staticParameters", - schema, - source: null, - }); - expect(result.success).toBe(true); - }); - - test("parses braintrust.staticParameters with source omitted", () => { - const result = serializedParametersContainerSchema.safeParse({ - type: "braintrust.staticParameters", - schema, - }); - expect(result.success).toBe(true); - }); -}); - -describe("evaluatorDefinitionSchema parameters nullish", () => { - test("allows parameters: null", () => { - const result = evaluatorDefinitionSchema.safeParse({ - parameters: null, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.parameters).toBeNull(); - } - }); - - test("allows parameters: undefined", () => { - const result = evaluatorDefinitionSchema.safeParse({ - parameters: undefined, - }); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.parameters).toBeUndefined(); - } - }); - - test("allows parameters omitted entirely", () => { - const result = evaluatorDefinitionSchema.safeParse({}); - expect(result.success).toBe(true); - if (result.success) { - expect(result.data.parameters).toBeUndefined(); - } - }); -}); diff --git a/js/src/zod/zod-v4-serialization.test.ts b/js/src/zod/zod-v4-serialization.test.ts deleted file mode 100644 index 9ba907e01..000000000 --- a/js/src/zod/zod-v4-serialization.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -/** - * Tests for zod v4 compatibility with serializeEvalParametersToStaticParametersSchema - * - * This file tests makeEvalParametersSchema specifically with zod v4. - * See zod-v3-serialization.test.ts for zod v3 specific tests. - */ - -import { test, describe, beforeEach, expect } from "vitest"; -import * as zodModule from "zod"; -import { z } from "zod"; -import { - EXPECTED_STRING_SCHEMA, - EXPECTED_NUMBER_SCHEMA, - EXPECTED_OBJECT_SCHEMA, - EXPECTED_ENUM_SCHEMA, - EXPECTED_ARRAY_SCHEMA, -} from "./zod-serialization-test-shared"; -import { serializeEvalParametersToStaticParametersSchema } from "../framework2"; -// Detect which zod version is installed by checking for v4-specific properties -function getInstalledZodVersion(): 3 | 4 { - const testSchema = zodModule.z.string(); - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return "_zod" in (testSchema as any) ? 4 : 3; -} -describe("makeEvalParametersSchema with Zod v4", () => { - beforeEach(() => { - const version = getInstalledZodVersion(); - expect(version).toBe(4); - }); - - test("string schema serializes correctly", () => { - const parameters = { - instructions: z - .string() - .describe("The instructions for the agent") - .default("You are a helpful assistant."), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - expect(result.instructions).toBeDefined(); - expect(result.instructions.type).toBe("data"); - expect( - result.instructions.type === "data" && result.instructions.schema, - ).toStrictEqual(EXPECTED_STRING_SCHEMA); - expect(result.instructions.description).toBe( - "The instructions for the agent", - ); - expect(result.instructions.default).toBe("You are a helpful assistant."); - }); - - test("number schema serializes correctly", () => { - const parameters = { - temperature: z - .number() - .min(0) - .max(2) - .describe("Temperature for LLM") - .default(0.7), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.temperature.type).toBe("data"); - expect( - result.temperature.type === "data" && result.temperature.schema, - ).toStrictEqual(EXPECTED_NUMBER_SCHEMA); - expect(result.temperature.description).toBe("Temperature for LLM"); - expect(result.temperature.default).toBe(0.7); - }); - - test("object schema serializes correctly", () => { - const parameters = { - config: z - .object({ - model: z.string(), - maxTokens: z.number().optional(), - }) - .describe("Configuration object"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.config.type).toBe("data"); - expect(result.config.type === "data" && result.config.schema).toStrictEqual( - EXPECTED_OBJECT_SCHEMA, - ); - expect(result.config.description).toBe("Configuration object"); - }); - - test("enum schema serializes correctly", () => { - const parameters = { - mode: z - .enum(["fast", "accurate", "balanced"]) - .describe("Processing mode") - .default("balanced"), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.mode.type).toBe("data"); - expect(result.mode.type === "data" && result.mode.schema).toStrictEqual( - EXPECTED_ENUM_SCHEMA, - ); - expect(result.mode.description).toBe("Processing mode"); - expect(result.mode.default).toBe("balanced"); - }); - - test("array schema serializes correctly", () => { - const parameters = { - tags: z - .array(z.string()) - .describe("Tags for filtering") - .default(["default"]), - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.tags.type).toBe("data"); - expect(result.tags.type === "data" && result.tags.schema).toStrictEqual( - EXPECTED_ARRAY_SCHEMA, - ); - expect(result.tags.description).toBe("Tags for filtering"); - expect(result.tags.default).toEqual(["default"]); - }); - - test("model parameter serializes correctly", () => { - const parameters = { - model: { - type: "model" as const, - description: "Model to use", - default: "gpt-5-mini", - }, - }; - - const result = serializeEvalParametersToStaticParametersSchema(parameters); - - expect(result.model.type).toBe("model"); - expect(result.model.description).toBe("Model to use"); - expect(result.model.default).toBe("gpt-5-mini"); - }); -}); diff --git a/js/tests/api-compatibility/api-compatibility.test.ts b/js/tests/api-compatibility/api-compatibility.test.ts index 99768dd48..2fd785357 100644 --- a/js/tests/api-compatibility/api-compatibility.test.ts +++ b/js/tests/api-compatibility/api-compatibility.test.ts @@ -20,7 +20,7 @@ import type { Options } from "tsup"; * ## How It Works * * 1. Downloads the latest published version from npm - * 2. Extracts the .d.ts files for each entrypoint (main, browser, dev, util) + * 2. Extracts the .d.ts files for each entrypoint (main, browser, util) * 3. Parses both published and current .d.ts files using TypeScript Compiler API * 4. Compares exported symbols (functions, classes, interfaces, types, etc.) * 5. Fails if breaking changes are detected in non-major version bumps @@ -100,7 +100,6 @@ function getEntrypointName(entryFile: string, outDir: string): string { if (entryFile.includes("src/browser/index.ts")) return "browser"; if (entryFile.includes("src/edge-light/index.ts")) return "edge-light"; if (entryFile.includes("src/workerd/index.ts")) return "workerd"; - if (entryFile.includes("dev/index.ts")) return "dev"; if (entryFile.includes("util/index.ts")) return "util"; // Default to basename @@ -152,6 +151,45 @@ function getVersionBumpType( return "none"; } +function getChangesetBumpType( + packageName: string, +): "major" | "minor" | "patch" | null { + const changesetDir = path.join(__dirname, "..", "..", "..", ".changeset"); + if (!fs.existsSync(changesetDir)) { + return null; + } + + const bumpPriority = { patch: 1, minor: 2, major: 3 } as const; + let highestBump: "major" | "minor" | "patch" | null = null; + const escapedPackageName = packageName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const packageBumpPattern = new RegExp( + `^\\s*["']?${escapedPackageName}["']?\\s*:\\s*(major|minor|patch)\\s*$`, + "m", + ); + + for (const file of fs.readdirSync(changesetDir)) { + if (!file.endsWith(".md")) { + continue; + } + + const contents = fs.readFileSync(path.join(changesetDir, file), "utf8"); + const frontmatter = /^---\r?\n([\s\S]*?)\r?\n---/.exec(contents)?.[1]; + const bump = frontmatter?.match(packageBumpPattern)?.[1] as + | "major" + | "minor" + | "patch" + | undefined; + if ( + bump && + (!highestBump || bumpPriority[bump] > bumpPriority[highestBump]) + ) { + highestBump = bump; + } + } + + return highestBump; +} + /** * Extracts exports and their signatures from a .d.ts file. * For bulk exports (export { A, B, C }), looks up the actual declaration @@ -2857,13 +2895,21 @@ describe("API Compatibility", () => { // Determine version bump type const publishedVersionInfo = parseVersion(publishedVersion); const currentVersionInfo = parseVersion(currentVersion); - versionBumpType = getVersionBumpType( + const packageVersionBumpType = getVersionBumpType( publishedVersionInfo, currentVersionInfo, ); + const changesetBumpType = getChangesetBumpType("braintrust"); + const bumpPriority = { none: 0, patch: 1, minor: 2, major: 3 } as const; + versionBumpType = + changesetBumpType && + bumpPriority[changesetBumpType] > bumpPriority[packageVersionBumpType] + ? changesetBumpType + : packageVersionBumpType; console.log(`Published version: ${publishedVersion}`); console.log(`Current version: ${currentVersion}`); + console.log(`Changeset bump type: ${changesetBumpType ?? "none"}`); console.log(`Version bump type: ${versionBumpType}`); // Create temp directory for downloaded package @@ -2902,6 +2948,56 @@ describe("API Compatibility", () => { expect(fs.existsSync(path.join(tempDir, "package"))).toBe(true); }); + test("keeps public declarations free of expanded Zod schema graphs", () => { + const declarationRoot = path.join(__dirname, "..", ".."); + const publicDeclarationPaths = [ + "dist/index.d.ts", + "dist/browser.d.ts", + "util/dist/index.d.ts", + ]; + + for (const declarationTypesPath of publicDeclarationPaths) { + const declarationPath = path.join(declarationRoot, declarationTypesPath); + const declaration = fs.readFileSync(declarationPath, "utf8"); + + expect(declaration).not.toMatch(/z\.infer { if (!publishedVersion) { console.log("Skipping test: No published version available"); diff --git a/js/tsup.config.ts b/js/tsup.config.ts index 14bca6b36..416ef9679 100644 --- a/js/tsup.config.ts +++ b/js/tsup.config.ts @@ -32,33 +32,6 @@ export default defineConfig([ define, clean: true, }, - { - entry: { cli: "src/cli/index.ts" }, - format: ["cjs"], - removeNodeProtocol: false, - outDir: "dist", - external: ["esbuild", "prettier", "typescript", "zod"], - // CLI doesn't need DTS - dts: false, - define, - clean: false, - }, - { - entry: ["dev/index.ts"], - format: ["cjs", "esm"], - outDir: "dev/dist", - removeNodeProtocol: false, - external: ["esbuild", "prettier", "typescript", "zod"], - dts: { - // Split DTS generation to reduce memory usage - compilerOptions: { - skipLibCheck: true, - }, - }, - splitting: true, - define, - clean: true, - }, { entry: ["util/index.ts"], format: ["cjs", "esm"], diff --git a/js/util/git_fields.ts b/js/util/git_fields.ts index 8eb06d625..482a43526 100644 --- a/js/util/git_fields.ts +++ b/js/util/git_fields.ts @@ -1,4 +1,4 @@ -import { GitMetadataSettingsType as GitMetadataSettings } from "./generated_types"; +import type { GitMetadataSettingsType as GitMetadataSettings } from "./generated_plain_types"; export function mergeGitMetadataSettings( s1: GitMetadataSettings, diff --git a/js/util/object.ts b/js/util/object.ts index 7ea593005..941c6264a 100644 --- a/js/util/object.ts +++ b/js/util/object.ts @@ -1,7 +1,7 @@ -import { +import type { AsyncScoringControlType as AsyncScoringControl, - type ObjectReferenceType, -} from "./generated_types"; + ObjectReferenceType, +} from "./generated_plain_types"; import { Source, ASYNC_SCORING_CONTROL_FIELD, diff --git a/js/util/span_identifier_v3.ts b/js/util/span_identifier_v3.ts index f057f798b..78a753d53 100644 --- a/js/util/span_identifier_v3.ts +++ b/js/util/span_identifier_v3.ts @@ -38,7 +38,8 @@ export enum SpanObjectTypeV3 { PLAYGROUND_LOGS = 3, } -export const spanObjectTypeV3EnumSchema = z.nativeEnum(SpanObjectTypeV3); +export const spanObjectTypeV3EnumSchema: z.ZodType = + z.nativeEnum(SpanObjectTypeV3); export function spanObjectTypeV3ToTypedString( objectType: SpanObjectTypeV3, @@ -81,7 +82,35 @@ const _INTERNAL_SPAN_COMPONENT_UUID_FIELDS_ID_TO_NAME: Record< [InternalSpanComponentUUIDFields.ROOT_SPAN_ID]: "root_span_id", }; -export const spanComponentsV3Schema = z +type SpanObjectMetadata = + | { + object_id?: string | null; + compute_object_metadata_args?: null; + } + | { + object_id?: null; + compute_object_metadata_args: Record; + }; + +type SpanRowIds = + | { + row_id: string; + span_id: string; + root_span_id: string; + } + | { + row_id?: null; + span_id?: null; + root_span_id?: null; + }; + +export type SpanComponentsV3Data = { + object_type: SpanObjectTypeV3; + propagated_event?: Record | null; +} & SpanObjectMetadata & + SpanRowIds; + +export const spanComponentsV3Schema: z.ZodType = z .object({ object_type: spanObjectTypeV3EnumSchema, // TODO(manu): We should have a more elaborate zod schema for @@ -118,8 +147,6 @@ export const spanComponentsV3Schema = z ]), ); -export type SpanComponentsV3Data = z.infer; - export class SpanComponentsV3 { constructor(public data: SpanComponentsV3Data) {} diff --git a/js/util/span_identifier_v4.ts b/js/util/span_identifier_v4.ts index e92e3eddf..771d72aec 100644 --- a/js/util/span_identifier_v4.ts +++ b/js/util/span_identifier_v4.ts @@ -5,6 +5,7 @@ import { SpanComponentsV3, SpanObjectTypeV3, spanObjectTypeV3EnumSchema, + type SpanComponentsV3Data, } from "./span_identifier_v3"; import { ParentExperimentIds, @@ -19,7 +20,7 @@ import { uint8ArrayToString, } from "./bytes"; import { z } from "zod/v3"; -import { InvokeFunctionType as InvokeFunctionRequest } from "./generated_types"; +import type { InvokeFunctionType as InvokeFunctionRequest } from "./generated_plain_types"; import { mergeDicts } from "./object_util"; const ENCODING_VERSION_NUMBER_V4 = 4; @@ -88,7 +89,9 @@ const FIELDS_ID_TO_NAME: Record = { [Fields.ROOT_SPAN_ID]: "root_span_id", }; -export const spanComponentsV4Schema = z +export type SpanComponentsV4Data = SpanComponentsV3Data; + +export const spanComponentsV4Schema: z.ZodType = z .object({ object_type: spanObjectTypeV3EnumSchema, propagated_event: z.record(z.unknown()).nullish(), @@ -122,8 +125,6 @@ export const spanComponentsV4Schema = z ]), ); -export type SpanComponentsV4Data = z.infer; - export class SpanComponentsV4 { constructor(public data: SpanComponentsV4Data) {} diff --git a/knip.jsonc b/knip.jsonc index 40f840f2d..202457490 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -55,7 +55,6 @@ "scripts/**/*.ts", "examples/**/*.{ts,tsx,js,jsx,mjs,cjs}", "smoke/**/*.{ts,tsx,js,jsx,mjs,cjs}", - "dev/**/*.ts", "util/**/*.ts", ], "project": [ @@ -64,7 +63,6 @@ "scripts/**/*.ts", "examples/**/*.{ts,tsx,js,jsx,mjs,cjs}", "smoke/**/*.{ts,tsx,js,jsx,mjs,cjs,d.ts}", - "dev/**/*.ts", "util/**/*.ts", ], }, diff --git a/package.json b/package.json index 78f725852..aa3b53f88 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,6 @@ "dotagents:sync": "dotagents sync", "changeset": "changeset", "playground": "turbo run playground --filter=\"braintrust\"", - "playground:cli:push": "turbo run playground:cli:push --filter=\"braintrust\"", - "playground:cli:eval": "turbo run playground:cli:eval --filter=\"braintrust\"", "formatting": "prettier --check .", "lint": "turbo run lint", "fix:formatting": "prettier --write .", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 79b23cd1e..c6deed80a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -316,9 +316,6 @@ importers: js: dependencies: - '@next/env': - specifier: ^14.2.3 - version: 14.2.3 '@vercel/functions': specifier: ^1.0.2 version: 1.0.2 @@ -331,24 +328,12 @@ importers: ajv: specifier: ^8.20.0 version: 8.20.0 - argparse: - specifier: ^2.0.1 - version: 2.0.1 astring: specifier: ^1.9.0 version: 1.9.0 cjs-module-lexer: specifier: ^2.2.0 version: 2.2.0 - cli-progress: - specifier: ^3.12.0 - version: 3.12.0 - cli-table3: - specifier: ^0.6.5 - version: 0.6.5 - cors: - specifier: ^2.8.5 - version: 2.8.5 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -361,36 +346,21 @@ importers: eventsource-parser: specifier: ^1.1.2 version: 1.1.2 - express: - specifier: ^5.2.1 - version: 5.2.1 - http-errors: - specifier: ^2.0.0 - version: 2.0.1 meriyah: specifier: ^6.1.4 version: 6.1.4 - minimatch: - specifier: ^10.2.5 - version: 10.2.5 module-details-from-path: specifier: ^1.0.4 version: 1.0.4 mustache: specifier: ^4.2.0 version: 4.2.0 - pluralize: - specifier: ^8.0.0 - version: 8.0.0 semifies: specifier: ^1.0.0 version: 1.0.0 source-map: specifier: ^0.7.4 version: 0.7.6 - termi-link: - specifier: ^1.0.1 - version: 1.1.0 unplugin: specifier: ^2.3.5 version: 2.3.11 @@ -404,39 +374,18 @@ importers: '@anthropic-ai/sdk': specifier: ^0.60.0 version: 0.60.0 - '@nodelib/fs.walk': - specifier: ^1.2.8 - version: 1.2.8 - '@types/argparse': - specifier: ^2.0.14 - version: 2.0.14 '@types/async': specifier: ^3.2.24 version: 3.2.24 - '@types/cli-progress': - specifier: ^3.11.5 - version: 3.11.5 - '@types/cors': - specifier: ^2.8.17 - version: 2.8.17 '@types/esquery': specifier: ^1.5.4 version: 1.5.4 - '@types/express': - specifier: ^5.0.0 - version: 5.0.1 - '@types/http-errors': - specifier: ^2.0.4 - version: 2.0.4 '@types/mustache': specifier: ^4.2.5 version: 4.2.5 '@types/node': specifier: ^20.10.5 version: 20.19.40 - '@types/pluralize': - specifier: ^0.0.30 - version: 0.0.30 '@typescript-eslint/eslint-plugin': specifier: ^8.49.0 version: 8.50.0(@typescript-eslint/parser@8.50.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.4.4))(eslint@9.39.4(jiti@2.6.1))(typescript@5.4.4) @@ -686,10 +635,6 @@ packages: resolution: {integrity: sha512-EYlRokl8szrP9Z25qT5aepMdBjzBvHF9ZEhzIiUBc9guz/T31EqRgvD0QSgZcpE93xiwrr+OkB4nz0BZyF6fSA==} engines: {node: '>= 20.12.0'} - '@colors/colors@1.5.0': - resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} - engines: {node: '>=0.1.90'} - '@emnapi/core@1.11.1': resolution: {integrity: sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==} @@ -1111,9 +1056,6 @@ packages: '@emnapi/core': ^1.7.1 '@emnapi/runtime': ^1.7.1 - '@next/env@14.2.3': - resolution: {integrity: sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA==} - '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -1779,27 +1721,12 @@ packages: '@tybys/wasm-util@0.10.3': resolution: {integrity: sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==} - '@types/argparse@2.0.14': - resolution: {integrity: sha512-jJ6NMs9rXQ0rsqNt3TL4Elcwhd6wygo3lJOVoiHzURD34vsCcAlw443uGu4PXTtEmMF7sYKoadTCLXNmuJuQGw==} - '@types/async@3.2.24': resolution: {integrity: sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==} - '@types/body-parser@1.19.5': - resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} - '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} - '@types/cli-progress@3.11.5': - resolution: {integrity: sha512-D4PbNRbviKyppS5ivBGyFO29POlySLmA2HyUFE4p5QGazAMM3CwkKWcvTl8gvElSuxRh6FPKL8XmidX873ou4g==} - - '@types/connect@3.4.38': - resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} - - '@types/cors@2.8.17': - resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} - '@types/deep-eql@4.0.2': resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} @@ -1818,21 +1745,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} - '@types/express-serve-static-core@5.0.6': - resolution: {integrity: sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA==} - - '@types/express@5.0.1': - resolution: {integrity: sha512-UZUw8vjpWFXuDnjFTh7/5c2TWDlQqeXHi6hcN7F2XSVT5P+WmUnnbFS3KA6Jnc6IsEqI2qCVu2bK0R0J4A8ZQQ==} - - '@types/http-errors@2.0.4': - resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} - '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} - '@types/mime@1.3.5': - resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} - '@types/mustache@4.2.5': resolution: {integrity: sha512-PLwiVvTBg59tGFL/8VpcGvqOu3L4OuveNvPi0EYbWchRdEVP++yRUXJPFl+CApKEq13017/4Nf7aQ5lTtHUNsA==} @@ -1851,24 +1766,9 @@ packages: '@types/nunjucks@3.2.6': resolution: {integrity: sha512-pHiGtf83na1nCzliuAdq8GowYiXvH5l931xZ0YEHaLMNFgynpEqx+IPStlu7UaDkehfvl01e4x/9Tpwhy7Ue3w==} - '@types/pluralize@0.0.30': - resolution: {integrity: sha512-kVww6xZrW/db5BR9OqiT71J9huRdQ+z/r+LbDuT7/EK50mCmj5FoaIARnVv0rvjUS/YpDox0cDU9lpQT011VBA==} - - '@types/qs@6.9.18': - resolution: {integrity: sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==} - - '@types/range-parser@1.2.7': - resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} - '@types/retry@0.12.0': resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} - '@types/send@0.17.4': - resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} - - '@types/serve-static@1.15.7': - resolution: {integrity: sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==} - '@types/set-cookie-parser@2.4.10': resolution: {integrity: sha512-GGmQVGpQWUe5qglJozEjZV/5dyxbOOZ0LHe/lqyWssB88Y4svNfst0uqBVscdDeIKl5Jy5+aPSvy7mI9tYRguw==} @@ -2154,10 +2054,6 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - balanced-match@4.0.4: - resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} - engines: {node: 18 || 20 || >=22} - base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -2179,10 +2075,6 @@ packages: brace-expansion@2.1.4: resolution: {integrity: sha512-hGfVzPxthbf3+2yjg/RBs60cB0FhqBS/zvdV/4wn4/BmN0bNMMHPc4V/BbFieqf1TKAGGAHnY4eSjajCl0f2Xg==} - brace-expansion@5.0.9: - resolution: {integrity: sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==} - engines: {node: 20 || >=22} - braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} @@ -2262,14 +2154,6 @@ packages: resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} engines: {node: '>=18'} - cli-progress@3.12.0: - resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} - engines: {node: '>=4'} - - cli-table3@0.6.5: - resolution: {integrity: sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==} - engines: {node: 10.* || >= 12.*} - cli-truncate@5.1.1: resolution: {integrity: sha512-SroPvNHxUnk+vIW/dOSfNqdy1sPEFkrTk6TUtqLCnBlo3N7TNYYkzzN7uSD6+jVjrdO4+p8nH7JzH6cIvUem6A==} engines: {node: '>=20'} @@ -3201,10 +3085,6 @@ packages: resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} engines: {node: '>=18'} - minimatch@10.2.5: - resolution: {integrity: sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==} - engines: {node: 18 || 20 || >=22} - minimatch@3.1.5: resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} @@ -3506,10 +3386,6 @@ packages: pkg-types@1.3.1: resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} - pluralize@8.0.0: - resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} - engines: {node: '>=4'} - postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -3853,10 +3729,6 @@ packages: resolution: {integrity: sha512-wK0Ri4fOGjv/XPy8SBHZChl8CM7uMc5VML7SqiQ0zG7+J5Vr+RMQDoHa2CNT6KHUnTGIXH34UDMkPzAUyapBZg==} engines: {node: '>=8'} - termi-link@1.1.0: - resolution: {integrity: sha512-2qSN6TnomHgVLtk+htSWbaYs4Rd2MH/RU7VpHTy6MBstyNyWbM4yKd1DCYpE3fDg8dmGWojXCngNi/MHCzGuAA==} - engines: {node: '>=12'} - terser-webpack-plugin@5.6.0: resolution: {integrity: sha512-Eum+5ajkaOhf5KbM26osvv21kLD7BaGqQ1UA4Ami4arYwylmGUQTgHFpHDdmJod1q4QXa66p0to/FBKID+J1vA==} engines: {node: '>= 10.13.0'} @@ -4558,9 +4430,6 @@ snapshots: fast-wrap-ansi: 0.2.0 sisteransi: 1.0.5 - '@colors/colors@1.5.0': - optional: true - '@emnapi/core@1.11.1': dependencies: '@emnapi/wasi-threads': 1.2.2 @@ -4986,8 +4855,6 @@ snapshots: '@tybys/wasm-util': 0.10.3 optional: true - '@next/env@14.2.3': {} - '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -5548,32 +5415,13 @@ snapshots: tslib: 2.8.1 optional: true - '@types/argparse@2.0.14': {} - '@types/async@3.2.24': {} - '@types/body-parser@1.19.5': - dependencies: - '@types/connect': 3.4.38 - '@types/node': 22.19.1 - '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 assertion-error: 2.0.1 - '@types/cli-progress@3.11.5': - dependencies: - '@types/node': 22.19.1 - - '@types/connect@3.4.38': - dependencies: - '@types/node': 22.19.1 - - '@types/cors@2.8.17': - dependencies: - '@types/node': 22.19.1 - '@types/deep-eql@4.0.2': {} '@types/eslint-scope@3.7.7': @@ -5594,25 +5442,8 @@ snapshots: '@types/estree@1.0.9': {} - '@types/express-serve-static-core@5.0.6': - dependencies: - '@types/node': 22.19.1 - '@types/qs': 6.9.18 - '@types/range-parser': 1.2.7 - '@types/send': 0.17.4 - - '@types/express@5.0.1': - dependencies: - '@types/body-parser': 1.19.5 - '@types/express-serve-static-core': 5.0.6 - '@types/serve-static': 1.15.7 - - '@types/http-errors@2.0.4': {} - '@types/json-schema@7.0.15': {} - '@types/mime@1.3.5': {} - '@types/mustache@4.2.5': {} '@types/node@12.20.55': {} @@ -5631,25 +5462,8 @@ snapshots: '@types/nunjucks@3.2.6': {} - '@types/pluralize@0.0.30': {} - - '@types/qs@6.9.18': {} - - '@types/range-parser@1.2.7': {} - '@types/retry@0.12.0': {} - '@types/send@0.17.4': - dependencies: - '@types/mime': 1.3.5 - '@types/node': 22.19.1 - - '@types/serve-static@1.15.7': - dependencies: - '@types/http-errors': 2.0.4 - '@types/node': 22.19.1 - '@types/send': 0.17.4 - '@types/set-cookie-parser@2.4.10': dependencies: '@types/node': 22.19.1 @@ -5914,6 +5728,7 @@ snapshots: dependencies: mime-types: 3.0.1 negotiator: 1.0.0 + optional: true acorn-import-attributes@1.9.5(acorn@8.16.0): dependencies: @@ -6007,8 +5822,6 @@ snapshots: balanced-match@1.0.2: {} - balanced-match@4.0.4: {} - base64-js@1.5.1: {} baseline-browser-mapping@2.9.14: {} @@ -6030,6 +5843,7 @@ snapshots: type-is: 2.1.0 transitivePeerDependencies: - supports-color + optional: true brace-expansion@1.1.18: dependencies: @@ -6040,10 +5854,6 @@ snapshots: dependencies: balanced-match: 1.0.2 - brace-expansion@5.0.9: - dependencies: - balanced-match: 4.0.4 - braces@3.0.3: dependencies: fill-range: 7.1.1 @@ -6063,7 +5873,8 @@ snapshots: esbuild: 0.28.1 load-tsconfig: 0.2.5 - bytes@3.1.2: {} + bytes@3.1.2: + optional: true cac@6.7.14: {} @@ -6071,11 +5882,13 @@ snapshots: dependencies: es-errors: 1.3.0 function-bind: 1.1.2 + optional: true call-bound@1.0.4: dependencies: call-bind-apply-helpers: 1.0.2 get-intrinsic: 1.3.0 + optional: true callsites@3.1.0: {} @@ -6108,16 +5921,6 @@ snapshots: dependencies: restore-cursor: 5.1.0 - cli-progress@3.12.0: - dependencies: - string-width: 4.2.3 - - cli-table3@0.6.5: - dependencies: - string-width: 4.2.3 - optionalDependencies: - '@colors/colors': 1.5.0 - cli-truncate@5.1.1: dependencies: slice-ansi: 7.1.2 @@ -6162,16 +5965,21 @@ snapshots: content-disposition@1.0.0: dependencies: safe-buffer: 5.2.1 + optional: true - content-type@1.0.5: {} + content-type@1.0.5: + optional: true - content-type@2.0.0: {} + content-type@2.0.0: + optional: true convert-source-map@2.0.0: {} - cookie-signature@1.2.2: {} + cookie-signature@1.2.2: + optional: true - cookie@0.7.2: {} + cookie@0.7.2: + optional: true cookie@1.1.1: {} @@ -6179,6 +5987,7 @@ snapshots: dependencies: object-assign: 4.1.1 vary: 1.1.2 + optional: true cross-env@7.0.3: dependencies: @@ -6200,7 +6009,8 @@ snapshots: deep-is@0.1.4: {} - depd@2.0.0: {} + depd@2.0.0: + optional: true detect-indent@6.1.0: {} @@ -6219,10 +6029,12 @@ snapshots: call-bind-apply-helpers: 1.0.2 es-errors: 1.3.0 gopd: 1.2.0 + optional: true eastasianwidth@0.2.0: {} - ee-first@1.1.1: {} + ee-first@1.1.1: + optional: true electron-to-chromium@1.5.267: {} @@ -6232,7 +6044,8 @@ snapshots: emoji-regex@9.2.2: {} - encodeurl@2.0.0: {} + encodeurl@2.0.0: + optional: true encoding@0.1.13: dependencies: @@ -6251,15 +6064,18 @@ snapshots: environment@1.1.0: {} - es-define-property@1.0.1: {} + es-define-property@1.0.1: + optional: true - es-errors@1.3.0: {} + es-errors@1.3.0: + optional: true es-module-lexer@2.1.0: {} es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 + optional: true esbuild@0.28.1: optionalDependencies: @@ -6292,7 +6108,8 @@ snapshots: escalade@3.2.0: {} - escape-html@1.0.3: {} + escape-html@1.0.3: + optional: true escape-string-regexp@4.0.0: {} @@ -6420,7 +6237,8 @@ snapshots: esutils@2.0.3: {} - etag@1.8.1: {} + etag@1.8.1: + optional: true event-target-shim@5.0.1: {} @@ -6479,6 +6297,7 @@ snapshots: vary: 1.1.2 transitivePeerDependencies: - supports-color + optional: true extendable-error@0.1.7: {} @@ -6538,6 +6357,7 @@ snapshots: statuses: 2.0.2 transitivePeerDependencies: - supports-color + optional: true find-up@4.1.0: dependencies: @@ -6571,9 +6391,11 @@ snapshots: dependencies: fd-package-json: 2.0.0 - forwarded@0.2.0: {} + forwarded@0.2.0: + optional: true - fresh@2.0.0: {} + fresh@2.0.0: + optional: true fs-extra@7.0.1: dependencies: @@ -6594,7 +6416,8 @@ snapshots: fsevents@2.3.3: optional: true - function-bind@1.1.2: {} + function-bind@1.1.2: + optional: true get-caller-file@2.0.5: {} @@ -6612,11 +6435,13 @@ snapshots: has-symbols: 1.1.0 hasown: 2.0.3 math-intrinsics: 1.1.0 + optional: true get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 es-object-atoms: 1.1.1 + optional: true get-tsconfig@4.13.0: dependencies: @@ -6669,7 +6494,8 @@ snapshots: globrex@0.1.2: {} - gopd@1.2.0: {} + gopd@1.2.0: + optional: true graceful-fs@4.2.11: {} @@ -6686,11 +6512,13 @@ snapshots: has-flag@4.0.0: {} - has-symbols@1.1.0: {} + has-symbols@1.1.0: + optional: true hasown@2.0.3: dependencies: function-bind: 1.1.2 + optional: true headers-polyfill@5.0.1: dependencies: @@ -6709,6 +6537,7 @@ snapshots: setprototypeof: 1.2.0 statuses: 2.0.2 toidentifier: 1.0.1 + optional: true human-id@4.1.3: {} @@ -6751,7 +6580,8 @@ snapshots: ip-address@10.4.0: optional: true - ipaddr.js@1.9.1: {} + ipaddr.js@1.9.1: + optional: true is-extglob@2.1.1: {} @@ -6769,7 +6599,8 @@ snapshots: is-number@7.0.0: {} - is-promise@4.0.0: {} + is-promise@4.0.0: + optional: true is-subdir@1.2.0: dependencies: @@ -6983,9 +6814,11 @@ snapshots: marked@4.3.0: {} - math-intrinsics@1.1.0: {} + math-intrinsics@1.1.0: + optional: true - media-typer@1.1.0: {} + media-typer@1.1.0: + optional: true memfs@4.51.1: dependencies: @@ -6996,7 +6829,8 @@ snapshots: tree-dump: 1.1.0(tslib@2.8.1) tslib: 2.8.1 - merge-descriptors@2.0.0: {} + merge-descriptors@2.0.0: + optional: true merge-stream@2.0.0: {} @@ -7014,13 +6848,10 @@ snapshots: mime-types@3.0.1: dependencies: mime-db: 1.54.0 + optional: true mimic-function@5.0.1: {} - minimatch@10.2.5: - dependencies: - brace-expansion: 5.0.9 - minimatch@3.1.5: dependencies: brace-expansion: 1.1.18 @@ -7171,7 +7002,8 @@ snapshots: natural-compare@1.4.0: {} - negotiator@1.0.0: {} + negotiator@1.0.0: + optional: true neo-async@2.6.2: {} @@ -7206,13 +7038,15 @@ snapshots: object-assign@4.1.1: {} - object-inspect@1.13.4: {} + object-inspect@1.13.4: + optional: true obug@2.1.1: {} on-finished@2.4.1: dependencies: ee-first: 1.1.1 + optional: true once@1.4.0: dependencies: @@ -7327,7 +7161,8 @@ snapshots: dependencies: callsites: 3.1.0 - parseurl@1.3.3: {} + parseurl@1.3.3: + optional: true path-exists@4.0.0: {} @@ -7342,7 +7177,8 @@ snapshots: path-to-regexp@6.3.0: {} - path-to-regexp@8.4.0: {} + path-to-regexp@8.4.0: + optional: true path-type@4.0.0: {} @@ -7369,8 +7205,6 @@ snapshots: mlly: 1.8.0 pathe: 2.0.3 - pluralize@8.0.0: {} - postcss-load-config@6.0.1(jiti@2.6.1)(postcss@8.5.19)(tsx@4.21.0)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 @@ -7414,18 +7248,21 @@ snapshots: dependencies: forwarded: 0.2.0 ipaddr.js: 1.9.1 + optional: true punycode@2.3.1: {} qs@6.15.2: dependencies: side-channel: 1.1.0 + optional: true quansync@0.2.11: {} queue-microtask@1.2.3: {} - range-parser@1.2.1: {} + range-parser@1.2.1: + optional: true raw-body@3.0.2: dependencies: @@ -7433,6 +7270,7 @@ snapshots: http-errors: 2.0.1 iconv-lite: 0.7.2 unpipe: 1.0.0 + optional: true react-dom@19.2.4(react@19.2.4): dependencies: @@ -7536,6 +7374,7 @@ snapshots: path-to-regexp: 8.4.0 transitivePeerDependencies: - supports-color + optional: true run-parallel@1.2.0: dependencies: @@ -7545,7 +7384,8 @@ snapshots: dependencies: tslib: 2.8.1 - safe-buffer@5.2.1: {} + safe-buffer@5.2.1: + optional: true safer-buffer@2.1.2: {} @@ -7578,6 +7418,7 @@ snapshots: statuses: 2.0.2 transitivePeerDependencies: - supports-color + optional: true serve-static@2.2.0: dependencies: @@ -7587,10 +7428,12 @@ snapshots: send: 1.2.0 transitivePeerDependencies: - supports-color + optional: true set-cookie-parser@3.1.0: {} - setprototypeof@1.2.0: {} + setprototypeof@1.2.0: + optional: true shebang-command@2.0.0: dependencies: @@ -7609,6 +7452,7 @@ snapshots: dependencies: es-errors: 1.3.0 object-inspect: 1.13.4 + optional: true side-channel-map@1.0.1: dependencies: @@ -7616,6 +7460,7 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.3.0 object-inspect: 1.13.4 + optional: true side-channel-weakmap@1.0.2: dependencies: @@ -7624,6 +7469,7 @@ snapshots: get-intrinsic: 1.3.0 object-inspect: 1.13.4 side-channel-map: 1.0.1 + optional: true side-channel@1.1.0: dependencies: @@ -7632,6 +7478,7 @@ snapshots: side-channel-list: 1.0.0 side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + optional: true siginfo@2.0.0: {} @@ -7757,8 +7604,6 @@ snapshots: term-size@2.2.1: {} - termi-link@1.1.0: {} - terser-webpack-plugin@5.6.0(@swc/core@1.15.8)(esbuild@0.28.1)(postcss@8.5.19)(webpack@5.106.2(@swc/core@1.15.8)(esbuild@0.28.1)(postcss@8.5.19)): dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -7815,7 +7660,8 @@ snapshots: dependencies: is-number: 7.0.0 - toidentifier@1.0.1: {} + toidentifier@1.0.1: + optional: true tough-cookie@6.0.1: dependencies: @@ -7964,6 +7810,7 @@ snapshots: content-type: 2.0.0 media-typer: 1.1.0 mime-types: 3.0.1 + optional: true typedoc-plugin-markdown@3.17.1(typedoc@0.25.13(typescript@5.4.4)): dependencies: @@ -7999,7 +7846,8 @@ snapshots: universalify@0.1.2: {} - unpipe@1.0.0: {} + unpipe@1.0.0: + optional: true unplugin@2.3.11: dependencies: @@ -8022,7 +7870,8 @@ snapshots: uuid@11.1.1: {} - vary@1.1.2: {} + vary@1.1.2: + optional: true vite-tsconfig-paths@4.3.2(typescript@5.4.4)(vite@8.1.5(@types/node@20.19.40)(esbuild@0.28.1)(jiti@2.6.1)(terser@5.44.1)(tsx@4.21.0)(yaml@2.9.0)): dependencies: diff --git a/turbo.json b/turbo.json index 6e5da8d56..6afc75def 100644 --- a/turbo.json +++ b/turbo.json @@ -98,38 +98,6 @@ "HUGGINGFACE_API_KEY" ], "dependsOn": ["^build", "build"] - }, - "playground:cli:push": { - "cache": false, - "passThroughEnv": [ - "ANTHROPIC_API_KEY", - "BRAINTRUST_API_KEY", - "GEMINI_API_KEY", - "COHERE_API_KEY", - "CURSOR_API_KEY", - "GROQ_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - "MISTRAL_API_KEY", - "HUGGINGFACE_API_KEY" - ], - "dependsOn": ["^build", "build"] - }, - "playground:cli:eval": { - "cache": false, - "passThroughEnv": [ - "ANTHROPIC_API_KEY", - "BRAINTRUST_API_KEY", - "GEMINI_API_KEY", - "COHERE_API_KEY", - "CURSOR_API_KEY", - "GROQ_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - "MISTRAL_API_KEY", - "HUGGINGFACE_API_KEY" - ], - "dependsOn": ["^build", "build"] } } }