diff --git a/command-snapshot.json b/command-snapshot.json index 39455470..d71e6c8b 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -1,512 +1,887 @@ [ - { - "alias": [], - "command": "agent:activate", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "target-org", "version"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:create", - "flagAliases": [], - "flagChars": ["n", "o", "w"], - "flags": [ - "api-version", - "content-fields", - "data-category-ids", - "data-category-names", - "description", - "developer-name", - "flags-dir", - "index-mode", - "json", - "name", - "primary-index-field1", - "primary-index-field2", - "retriever-id", - "source-type", - "target-org", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:delete", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:add", - "flagAliases": [], - "flagChars": ["f", "i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "path", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:delete", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "file-id", "flags-dir", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:list", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "offset", "page-size", "status", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:get", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:list", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-version", "flags-dir", "json", "source-type", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:status", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "include-artifacts", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:update", - "flagAliases": [], - "flagChars": ["i", "n", "o"], - "flags": [ - "api-version", - "content-fields", - "data-category-rule", - "description", - "flags-dir", - "json", - "library-id", - "name", - "restrict-to-public-articles", - "retriever-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:upload", - "flagAliases": [], - "flagChars": ["f", "i", "o", "w"], - "flags": ["api-version", "file", "flags-dir", "json", "library-id", "target-org", "wait"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:create", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "name", "planner-id", "preview", "spec", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:deactivate", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:agent-spec", - "flagAliases": [], - "flagChars": ["o"], - "flags": [ - "agent-user", - "api-version", - "company-description", - "company-name", - "company-website", - "enrich-logs", - "flags-dir", - "force-overwrite", - "full-interview", - "grounding-context", - "json", - "max-topics", - "output-file", - "prompt-template", - "role", - "spec", - "target-org", - "tone", - "type" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:authoring-bundle", - "flagAliases": [], - "flagChars": ["d", "f", "n", "o"], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "force-overwrite", - "json", - "name", - "no-spec", - "output-dir", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:template", - "flagAliases": [], - "flagChars": ["f", "r", "s"], - "flags": ["agent-file", "agent-version", "api-version", "flags-dir", "json", "output-dir", "source-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:test-spec", - "flagAliases": [], - "flagChars": ["d", "f"], - "flags": ["flags-dir", "force-overwrite", "from-definition", "output-file", "test-runner"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:asset:list", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:asset:replace", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "assets", "assets-file", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:create", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": [ - "api-version", - "auth-type", - "client-id", - "client-secret", - "description", - "flags-dir", - "identity-provider", - "json", - "label", - "name", - "scope", - "server-url", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:delete", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "no-prompt", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:fetch", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:get", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:list", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-version", "flags-dir", "json", "label", "status", "target-org", "type"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:update", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": [ - "api-version", - "auth-type", - "client-id", - "client-secret", - "description", - "flags-dir", - "identity-provider", - "json", - "label", - "mcp-server-id", - "scope", - "server-url", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview", - "flagAliases": [], - "flagChars": ["d", "n", "o", "x"], - "flags": [ - "agent-json", - "apex-debug", - "api-name", - "api-version", - "authoring-bundle", - "context-variables", - "flags-dir", - "output-dir", - "target-org", - "use-live-actions" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:end", - "flagAliases": [], - "flagChars": ["n", "o", "p"], - "flags": [ - "all", - "api-name", - "api-version", - "authoring-bundle", - "flags-dir", - "json", - "no-prompt", - "session-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:send", - "flagAliases": [], - "flagChars": ["n", "o", "u"], - "flags": [ - "api-name", - "api-version", - "authoring-bundle", - "flags-dir", - "json", - "session-id", - "target-org", - "utterance" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:sessions", - "flagAliases": [], - "flagChars": [], - "flags": ["flags-dir", "json"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:start", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": [ - "agent-json", - "api-name", - "api-version", - "authoring-bundle", - "context-variables", - "flags-dir", - "json", - "simulate-actions", - "target-org", - "use-live-actions" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:publish:authoring-bundle", - "flagAliases": [], - "flagChars": ["n", "o", "v"], - "flags": ["api-name", "api-version", "concise", "flags-dir", "json", "skip-retrieve", "target-org", "verbose"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:create", - "flagAliases": [], - "flagChars": ["o"], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "force-overwrite", - "json", - "preview", - "spec", - "target-org", - "test-runner" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:list", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-version", "flags-dir", "json", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:results", - "flagAliases": [], - "flagChars": ["d", "i", "o"], - "flags": [ - "api-version", - "flags-dir", - "job-id", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "verbose" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:resume", - "flagAliases": [], - "flagChars": ["d", "i", "o", "r", "w"], - "flags": [ - "api-version", - "flags-dir", - "job-id", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "use-most-recent", - "verbose", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:run", - "flagAliases": [], - "flagChars": ["d", "n", "o", "w"], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "verbose", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:run-eval", - "flagAliases": [], - "flagChars": ["n", "o", "s"], - "flags": [ - "api-name", - "api-version", - "batch-size", - "flags-dir", - "json", - "no-normalize", - "result-format", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:delete", - "flagAliases": [], - "flagChars": ["a"], - "flags": ["agent", "flags-dir", "json", "no-prompt", "older-than", "session-id"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:list", - "flagAliases": [], - "flagChars": ["a"], - "flags": ["agent", "flags-dir", "json", "session-id", "since"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:read", - "flagAliases": [], - "flagChars": ["d", "f", "s", "t"], - "flags": ["dimension", "flags-dir", "format", "json", "session-id", "turn"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:validate:authoring-bundle", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "target-org"], - "plugin": "@salesforce/plugin-agent" - } -] + { + "alias": [], + "command": "agent:activate", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "target-org", + "version" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:create", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "w" + ], + "flags": [ + "api-version", + "content-fields", + "data-category-ids", + "data-category-names", + "description", + "developer-name", + "flags-dir", + "index-mode", + "json", + "name", + "primary-index-field1", + "primary-index-field2", + "retriever-id", + "source-type", + "target-org", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:delete", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:add", + "flagAliases": [], + "flagChars": [ + "f", + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "path", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:delete", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "file-id", + "flags-dir", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:list", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "offset", + "page-size", + "status", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:get", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:list", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "source-type", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:status", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "include-artifacts", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:update", + "flagAliases": [], + "flagChars": [ + "i", + "n", + "o" + ], + "flags": [ + "api-version", + "content-fields", + "data-category-rule", + "description", + "flags-dir", + "json", + "library-id", + "name", + "restrict-to-public-articles", + "retriever-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:upload", + "flagAliases": [], + "flagChars": [ + "f", + "i", + "o", + "w" + ], + "flags": [ + "api-version", + "file", + "flags-dir", + "json", + "library-id", + "target-org", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:create", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "name", + "planner-id", + "preview", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:deactivate", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:agent-spec", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "agent-user", + "api-version", + "company-description", + "company-name", + "company-website", + "enrich-logs", + "flags-dir", + "force-overwrite", + "full-interview", + "grounding-context", + "json", + "max-topics", + "output-file", + "prompt-template", + "role", + "spec", + "target-org", + "tone", + "type" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:authoring-bundle", + "flagAliases": [], + "flagChars": [ + "d", + "f", + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "force-overwrite", + "json", + "name", + "no-spec", + "output-dir", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:template", + "flagAliases": [], + "flagChars": [ + "f", + "r", + "s" + ], + "flags": [ + "agent-file", + "agent-version", + "api-version", + "flags-dir", + "json", + "output-dir", + "source-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:test-spec", + "flagAliases": [], + "flagChars": [ + "d", + "f" + ], + "flags": [ + "flags-dir", + "force-overwrite", + "from-definition", + "output-file", + "test-runner" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:asset:list", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:asset:replace", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "assets", + "assets-file", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:create", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-version", + "auth-type", + "client-id", + "client-secret", + "description", + "flags-dir", + "identity-provider", + "json", + "label", + "name", + "scope", + "server-url", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:delete", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "no-prompt", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:fetch", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:get", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:list", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "label", + "status", + "target-org", + "type" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:update", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "auth-type", + "client-id", + "client-secret", + "description", + "flags-dir", + "identity-provider", + "json", + "label", + "mcp-server-id", + "scope", + "server-url", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview", + "flagAliases": [], + "flagChars": [ + "d", + "n", + "o", + "x" + ], + "flags": [ + "agent-json", + "apex-debug", + "api-name", + "api-version", + "authoring-bundle", + "context-variables", + "flags-dir", + "output-dir", + "target-org", + "use-live-actions" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:end", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "p" + ], + "flags": [ + "all", + "api-name", + "api-version", + "authoring-bundle", + "flags-dir", + "json", + "no-prompt", + "session-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:send", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "u" + ], + "flags": [ + "api-name", + "api-version", + "authoring-bundle", + "flags-dir", + "json", + "session-id", + "target-org", + "utterance" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:sessions", + "flagAliases": [], + "flagChars": [], + "flags": [ + "flags-dir", + "json" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:start", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "agent-json", + "api-name", + "api-version", + "authoring-bundle", + "context-variables", + "flags-dir", + "json", + "simulate-actions", + "target-org", + "use-live-actions" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:publish:authoring-bundle", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "v" + ], + "flags": [ + "api-name", + "api-version", + "concise", + "flags-dir", + "json", + "skip-retrieve", + "target-org", + "verbose" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:create", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "agent-api-name", + "api-name", + "api-version", + "description", + "engine-type", + "flags-dir", + "json", + "label", + "lightning-type", + "new-version", + "output-dir", + "preview", + "spec", + "spec-schema", + "status", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:edit", + "flagAliases": [], + "flagChars": [], + "flags": [ + "activate", + "api-name", + "deactivate", + "flags-dir", + "json", + "output-dir", + "preview", + "status", + "version" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:run", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-name", + "api-version", + "data", + "file", + "flags-dir", + "json", + "scorer-version", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:create", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "force-overwrite", + "json", + "preview", + "spec", + "target-org", + "test-runner" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:list", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:results", + "flagAliases": [], + "flagChars": [ + "d", + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:resume", + "flagAliases": [], + "flagChars": [ + "d", + "i", + "o", + "r", + "w" + ], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "use-most-recent", + "verbose", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:run", + "flagAliases": [], + "flagChars": [ + "d", + "n", + "o", + "w" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:run-eval", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "s" + ], + "flags": [ + "api-name", + "api-version", + "batch-size", + "flags-dir", + "json", + "no-normalize", + "result-format", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:delete", + "flagAliases": [], + "flagChars": [ + "a" + ], + "flags": [ + "agent", + "flags-dir", + "json", + "no-prompt", + "older-than", + "session-id" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:list", + "flagAliases": [], + "flagChars": [ + "a" + ], + "flags": [ + "agent", + "flags-dir", + "json", + "session-id", + "since" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:read", + "flagAliases": [], + "flagChars": [ + "d", + "f", + "s", + "t" + ], + "flags": [ + "dimension", + "flags-dir", + "format", + "json", + "session-id", + "turn" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:validate:authoring-bundle", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + } +] \ No newline at end of file diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.create.md new file mode 100644 index 00000000..68844522 --- /dev/null +++ b/messages/agent.scorer.create.md @@ -0,0 +1,107 @@ +# summary + +Create an agent scorer definition using an interactive interview or a spec file. + +# description + +Creates an AiAgentScorerDefinition metadata XML file either interactively (prompting for each field) or from a YAML spec file. + +Run with no flags to start the interactive interview. The command prompts you for the scorer's lightning type, optional output labels, engine type, and agent associations. + +Alternatively, provide a --spec flag pointing to a YAML file that defines the scorer. This is useful for repeatable automation or when the scorer has many output values. + +Use --preview to see the generated XML without writing it to disk. + +# flags.api-name.summary + +API name of the scorer definition. + +# flags.agent-api-name.summary + +API name of the agent to associate with this scorer. + +# flags.lightning-type.summary + +Lightning type the scorer's value conforms to (for example, lightning__textType or lightning__numberType). + +# flags.label.summary + +Display label for the scorer version. + +# flags.description.summary + +Description of what this scorer evaluates. + +# flags.engine-type.summary + +Engine type for scoring (Manual or PromptTemplate). + +# flags.status.summary + +Initial status of the scorer version (Available or Draft). + +# flags.spec.summary + +Path to a scorer spec YAML file. Bypasses interactive prompts. + +# flags.spec-schema.summary + +Output the JSON Schema for the --spec YAML file and exit. + +# flags.new-version.summary + +Add a new version to an existing scorer instead of erroring. The new version is numbered one higher than the current highest; if a new prompt rubric is supplied, the prompt template's active version is updated too. + +# flags.output-dir.summary + +Output directory for the generated metadata XML files (scorer definition and prompt template). + +# flags.preview.summary + +Preview the generated XML without writing to disk. + +# examples + +- Show the JSON Schema for the spec YAML file: + + <%= config.bin %> <%= command.id %> --spec-schema + +- Create a scorer interactively: + + <%= config.bin %> <%= command.id %> + +- Create a scorer from a spec file: + + <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml + +- Preview the XML that would be generated: + + <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml --preview + +- Create a manual scorer with flags (non-interactive): + + <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning__textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available + +- Create a prompt-based scorer (generates both scorer definition and prompt template): + + <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning__textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent + +# error.missingRequiredFlags + +Missing required flags: %s. When using --json, all required flags must be provided. + +# error.invalidSpecYaml + +Could not parse the --spec file as YAML: %s + +# error.invalidSpecShape + +The --spec file must define a YAML object matching the scorer spec schema (see --spec-schema). Received: %s + +# error.noAgentsInOrg + +No agents found in the org. Deploy an agent first, or specify one with --agent-api-name. + +# error.scorerExists + +A scorer named '%s' already exists in this project. To refine it, add a new version with --new-version; to change a version's status use `sf agent scorer edit`; or use --preview to see the generated XML without writing. diff --git a/messages/agent.scorer.edit.md b/messages/agent.scorer.edit.md new file mode 100644 index 00000000..a3afdb05 --- /dev/null +++ b/messages/agent.scorer.edit.md @@ -0,0 +1,69 @@ +# summary + +Change the status or agent-association activation of a version of an existing agent scorer. + +# description + +Edits one version of a scorer in place: promote it (`--status Available`, so its rubric is served by default on the next run), archive it (`--status Archived`, so it can no longer be run), and/or turn its agent association on or off (`--activate` / `--deactivate`). Editing a scorer never authors content: a version's rubric is immutable once it exists — status and activation are the only fields that change. To add a new version with a refined rubric, use `sf agent scorer create --new-version`. + +This command edits the scorer's local metadata XML only; it does not require an org connection. Deploy the updated scorer definition afterward for the org to reflect the change. + +Platform activation rules (enforced on deploy): an active association (`--activate`) requires the version's status to be `Available`, and at most one version of a scorer may hold an active association. + +# flags.api-name.summary + +API name of the scorer to edit. Must match a scorer authored in this project's metadata. + +# flags.version.summary + +Version number to edit. + +# flags.status.summary + +New status for the version: Draft, Available (promote — served by default on the next run), or Archived (can no longer be run). + +# flags.activate.summary + +Activate the version's agent association (start scoring the associated agent's sessions). Requires the version's status to be Available. + +# flags.deactivate.summary + +Deactivate the version's agent association (stop scoring the associated agent's sessions). + +# flags.output-dir.summary + +Directory containing the scorer's metadata XML (where the scorer definition was authored). + +# flags.preview.summary + +Preview the resulting XML without writing to disk. + +# examples + +- Promote version 2 of a scorer to Available: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available + +- Archive version 1 so it can no longer be run: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 1 --status Archived + +- Promote a version and activate its agent association in a single command: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --activate + +- Deactivate the agent association on a version: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --deactivate + +- Preview a status change without writing to disk: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --preview + +# error.noChange + +Specify at least one change: --status, --activate, or --deactivate. + +# error.scorerNotFound + +No scorer '%s' was found at %s. Author it first with `sf agent scorer create`. diff --git a/messages/agent.scorer.run.md b/messages/agent.scorer.run.md new file mode 100644 index 00000000..378f8111 --- /dev/null +++ b/messages/agent.scorer.run.md @@ -0,0 +1,59 @@ +# summary + +Run an agent scorer against an STDM session and print its score. + +# description + +Runs a scorer that is already authored in your project metadata, referenced by its API name, against a single STDM (Session Trace Data Model) session, then prints the resulting score, outcome labels, and explanation. + +The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer create`. + +Provide the session either inline as a JSON string with --data, or as a path to a local JSON file with --file. Exactly one of the two is required. + +To help you hand-construct a valid session, run this command with --help: the full JSON Schema for the session object is printed under the --data flag. + +# flags.api-name.summary + +API name of the scorer to run. Must match a scorer authored in this project's metadata. + +# flags.scorer-version.summary + +Version number of the scorer to run. Omit to run the highest-numbered Available version; the command errors if none is Available. A Draft version must be selected explicitly; an Archived version cannot be run. + +# flags.data.summary + +Inline STDM session JSON to score. + +# flags.data.description + +Inline STDM session JSON to score. The value must be a JSON object matching the following JSON Schema (the shape of the scorer's Input:Session value): + +# flags.file.summary + +Path to a local JSON file containing the STDM session to score. + +# examples + +- Run a scorer against a session stored in a local file: + + <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --file ./session.json + +- Run a scorer against an inline session JSON string: + + <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --data '{"sessionState":{"sessionId":"1","startTimestamp":"2026-01-01T00:00:00Z","channel":"web"},"actors":[],"metrics":{"durationMs":0,"turns":0},"runs":[]}' + +- Show the session JSON Schema in the help output: + + <%= config.bin %> <%= command.id %> --help + +# error.invalidSessionJson + +Could not parse the STDM session as JSON: %s + +# error.invalidSessionShape + +The STDM session must be a JSON object matching the session schema (see --help). Received: %s + +# error.scorerRunFailed + +Scorer '%s' did not produce a valid score: %s. The command exits non-zero so a scripted loop won't treat a failed evaluation as a passing one; the full result (including any explanation) is in the error's "data" field when run with --json. diff --git a/package.json b/package.json index 17160dc7..9ec907eb 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,10 @@ "description": "Command to validate an Agent Script file.", "external": true }, + "scorer": { + "description": "Commands to create and manage agent scorers.", + "external": true + }, "adl": { "description": "Commands to manage Agentforce Data Libraries.", "external": true, diff --git a/schemas/agent-scorer-create.json b/schemas/agent-scorer-create.json new file mode 100644 index 00000000..65b33290 --- /dev/null +++ b/schemas/agent-scorer-create.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerCreateResult", + "definitions": { + "AgentScorerCreateResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "apiName": { + "type": "string" + }, + "contents": { + "type": "string" + }, + "promptTemplatePath": { + "type": "string" + } + }, + "required": [ + "path", + "apiName", + "contents" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/schemas/agent-scorer-edit.json b/schemas/agent-scorer-edit.json new file mode 100644 index 00000000..de6a2aea --- /dev/null +++ b/schemas/agent-scorer-edit.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerEditResult", + "definitions": { + "AgentScorerEditResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "apiName": { + "type": "string" + }, + "contents": { + "type": "string" + } + }, + "required": [ + "path", + "apiName", + "contents" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/schemas/agent-scorer-run.json b/schemas/agent-scorer-run.json new file mode 100644 index 00000000..9b5e9836 --- /dev/null +++ b/schemas/agent-scorer-run.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerRunResult", + "definitions": { + "AgentScorerRunResult": { + "type": "object", + "additionalProperties": false, + "properties": { + "scorerApiName": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "output": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "The score: one or more labels, or a typed value matching the scorer's lightning type." + }, + "explanation": { + "type": "string" + }, + "raw": { + "type": "string", + "description": "Raw engine output (for debugging / loose formats)." + }, + "error": { + "type": "string" + } + }, + "required": [ + "ok", + "scorerApiName" + ] + } + } +} \ No newline at end of file diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts new file mode 100644 index 00000000..60e25d22 --- /dev/null +++ b/src/commands/agent/scorer/create.ts @@ -0,0 +1,429 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join, resolve } from 'node:path'; +import { readFileSync, existsSync } from 'node:fs'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable } from '@salesforce/core'; +import { + Agent, + type ScorerSpec, + createScorerDefinition, + addScorerVersion, + labelToApiName, + scorerSpecJsonSchema, + type SupportedLightningType, + SUPPORTED_LIGHTNING_TYPES, + SCORER_API_NAME_MAX_LENGTH, + SCORER_API_NAME_PATTERN, + SCORER_ENGINE_TYPES, + SCORER_STATUSES, + SCORER_OUTCOME_TYPES, + SCORER_INPUT_SCOPES, +} from '@salesforce/agents'; +import { confirm, select, input as inquirerInput } from '@inquirer/prompts'; +import YAML from 'yaml'; +import { FlaggablePrompt, makeFlags, promptForFlag } from '../../../flags.js'; +import { theme } from '../../../inquirer-theme.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.create'); + +export type AgentScorerCreateResult = { + path: string; + apiName: string; + contents: string; + promptTemplatePath?: string; +}; + +const FLAGGABLE_PROMPTS = { + label: { + message: messages.getMessage('flags.label.summary'), + promptMessage: 'Scorer label (display name)', + validate: (d: string): boolean | string => d.length > 0 || 'Label cannot be empty', + required: true, + }, + 'api-name': { + message: messages.getMessage('flags.api-name.summary'), + promptMessage: 'Scorer API name', + validate: (d: string): boolean | string => { + if (!d.length) return 'API name cannot be empty'; + if (d.length > SCORER_API_NAME_MAX_LENGTH) return `API name cannot exceed ${SCORER_API_NAME_MAX_LENGTH} characters`; + if (!SCORER_API_NAME_PATTERN.test(d)) return 'Must start with letter, only alphanumerics and underscores'; + return true; + }, + required: true, + }, + 'lightning-type': { + message: messages.getMessage('flags.lightning-type.summary'), + promptMessage: 'Select the lightning type this scorer produces', + options: SUPPORTED_LIGHTNING_TYPES, + validate: (d: string): boolean | string => + (SUPPORTED_LIGHTNING_TYPES as readonly string[]).includes(d) || 'Invalid lightning type', + required: true, + }, + description: { + message: messages.getMessage('flags.description.summary'), + promptMessage: 'Description (optional, press Enter to skip)', + validate: (): boolean | string => true, + }, + 'engine-type': { + message: messages.getMessage('flags.engine-type.summary'), + promptMessage: 'Scoring engine type', + options: SCORER_ENGINE_TYPES, + validate: (d: string): boolean | string => + (SCORER_ENGINE_TYPES as readonly string[]).includes(d) || 'Invalid engine type', + required: true, + }, + status: { + message: messages.getMessage('flags.status.summary'), + promptMessage: 'Initial status', + options: SCORER_STATUSES, + validate: (d: string): boolean | string => (SCORER_STATUSES as readonly string[]).includes(d) || 'Invalid status', + default: 'Draft', + }, +} satisfies Record; + +type OutputEnumValueInput = { + value: string; + outcomeType: string; + isFallback: boolean; + isSystemFallback: boolean; +}; + +async function promptForSingleEnumValue(index: number): Promise { + const value = await promptForFlag({ + message: 'Output value name', + promptMessage: `Output value #${index + 1} (e.g., "Good", "Bad", "N/A")`, + validate: (d: string): boolean | string => d.length > 0 || 'Value cannot be empty', + }); + + const outcomeType = await promptForFlag({ + message: 'Outcome type', + promptMessage: 'Outcome type for this value', + options: SCORER_OUTCOME_TYPES, + validate: (d: string): boolean | string => + (SCORER_OUTCOME_TYPES as readonly string[]).includes(d) || 'Invalid', + }); + + const isFallback = await confirm({ + message: 'Is this the fallback value?', + default: index === 0, + theme, + }); + + const addMore = await confirm({ + message: 'Add another output value?', + default: index < 1, + theme, + }); + + return { value, outcomeType, isFallback, isSystemFallback: false, addMore }; +} + +async function promptForOutputEnumValues(): Promise { + const values: OutputEnumValueInput[] = []; + let addMore = true; + + while (addMore) { + // eslint-disable-next-line no-await-in-loop + const result = await promptForSingleEnumValue(values.length); + addMore = result.addMore; + values.push({ value: result.value, outcomeType: result.outcomeType, isFallback: result.isFallback, isSystemFallback: result.isSystemFallback }); + } + + return values; +} + +export default class AgentScorerCreate extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + ...makeFlags(FLAGGABLE_PROMPTS), + 'agent-api-name': Flags.string({ + summary: messages.getMessage('flags.agent-api-name.summary'), + }), + spec: Flags.file({ + summary: messages.getMessage('flags.spec.summary'), + exists: true, + }), + 'spec-schema': Flags.boolean({ + summary: messages.getMessage('flags.spec-schema.summary'), + default: false, + }), + 'new-version': Flags.boolean({ + summary: messages.getMessage('flags.new-version.summary'), + default: false, + }), + 'output-dir': Flags.directory({ + summary: messages.getMessage('flags.output-dir.summary'), + default: join('force-app', 'main', 'default'), + }), + preview: Flags.boolean({ + summary: messages.getMessage('flags.preview.summary'), + }), + }; + + // eslint-disable-next-line complexity + public async run(): Promise { + const { flags } = await this.parse(AgentScorerCreate); + + if (flags['spec-schema']) { + this.styledJSON(scorerSpecJsonSchema() as unknown as import('@salesforce/ts-types').AnyJson); + return { path: '', apiName: '', contents: '' }; + } + + const outputDir = resolve(flags['output-dir']); + + const connection = flags['target-org'].getConnection(flags['api-version']); + + const spec: ScorerSpec = flags.spec + ? this.parseSpec(readFileSync(resolve(flags.spec), 'utf8')) + : await this.runInteractiveInterview(flags, connection); + + const scorerFileName = `${spec.apiName}.aiAgentScorerDefinition-meta.xml`; + const scorerPath = join(outputDir, 'aiAgentScorerDefinitions', scorerFileName); + const exists = existsSync(scorerPath); + + // An existing scorer is never overwritten: refine it by adding a new version, which keeps the API name + // stable and the version history intact. Require --new-version so a re-run can't silently add a version + // (versions can't be deleted once deployed). This gate applies to --preview too, so a preview reflects + // the artifact the same flags would actually write. + if (exists && !flags['new-version']) { + throw messages.createError('error.scorerExists', [spec.apiName]); + } + + if (flags.preview) { + // Preview the exact artifact these flags would write: an appended version when the scorer already + // exists (--new-version), otherwise a fresh definition. + const result = exists + ? await addScorerVersion(spec, { outputDir, write: false }) + : await createScorerDefinition(spec, { outputDir, write: false }); + this.log('\n--- Scorer Definition (preview) ---\n'); + this.log(result.contents); + if (result.promptTemplateContents) { + this.log('\n--- Prompt Template (preview) ---\n'); + this.log(result.promptTemplateContents); + } + return { path: result.path, apiName: result.apiName, contents: result.contents, promptTemplatePath: result.promptTemplatePath }; + } + + if (exists) { + const added = await addScorerVersion(spec, { outputDir }); + this.log(`\nAdded version ${added.versionNumber} to scorer: ${added.path}`); + if (added.promptTemplatePath) { + this.log(`Updated prompt template: ${added.promptTemplatePath}`); + } + return { + path: added.path, + apiName: added.apiName, + contents: added.contents, + promptTemplatePath: added.promptTemplatePath, + }; + } + + const result = await createScorerDefinition(spec, { outputDir }); + this.log(`\nScorer definition written to: ${result.path}`); + if (result.promptTemplatePath) { + this.log(`Prompt template written to: ${result.promptTemplatePath}`); + } + + return { path: result.path, apiName: result.apiName, contents: result.contents, promptTemplatePath: result.promptTemplatePath }; + } + + // eslint-disable-next-line class-methods-use-this + private parseSpec(raw: string): ScorerSpec { + let parsed: unknown; + try { + parsed = YAML.parse(raw); + } catch (e) { + throw messages.createError('error.invalidSpecYaml', [(e as Error).message]); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw messages.createError('error.invalidSpecShape', [typeof parsed]); + } + return parsed as ScorerSpec; + } + + private async runInteractiveInterview( + flags: Record, + connection: ReturnType + ): Promise { + if (this.jsonEnabled()) { + const missing = Object.entries(FLAGGABLE_PROMPTS) + .filter(([key, p]) => 'required' in p && p.required && !flags[key]) + .map(([key]) => key); + if (!flags['agent-api-name']) missing.push('agent-api-name'); + if (missing.length) { + throw messages.createError('error.missingRequiredFlags', [missing.join(', ')]); + } + } + + this.log(); + this.styledHeader('Scorer Definition'); + + const label = (flags.label as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS.label)); + + const defaultApiName = labelToApiName(label); + const apiName = (flags['api-name'] as string) ?? (await inquirerInput({ + message: 'Scorer API name', + default: defaultApiName, + validate: FLAGGABLE_PROMPTS['api-name'].validate, + theme, + })); + + const description = + (flags.description as string) ?? + (this.jsonEnabled() ? undefined : await promptForFlag(FLAGGABLE_PROMPTS.description)); + const status = + (flags.status as string) ?? + (this.jsonEnabled() ? FLAGGABLE_PROMPTS.status.default : await promptForFlag(FLAGGABLE_PROMPTS.status)); + const lightningType = ((flags['lightning-type'] as SupportedLightningType) ?? + (await promptForFlag(FLAGGABLE_PROMPTS['lightning-type']))) as SupportedLightningType; + + this.log(); + this.styledHeader('Output Labels'); + const addLabels = this.jsonEnabled() + ? false + : await confirm({ + message: 'Add predefined output labels? (leave off for fully open-ended output)', + default: false, + theme, + }); + const outputEnumValues = addLabels ? await promptForOutputEnumValues() : undefined; + + const engineType = (flags['engine-type'] as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS['engine-type'])); + const engineConfig = await this.promptForEngineConfig(engineType); + const agentAssociation = await this.promptForAgentAssociationDetails( + connection, engineType, flags['agent-api-name'] as string | undefined + ); + + return { + apiName, + lightningType, + inputScope: 'Session', + label, + description: description || undefined, + engineType: engineType as ScorerSpec['engineType'], + promptContent: engineConfig.promptContent, + promptTemplateName: engineConfig.promptTemplateName, + status: status as ScorerSpec['status'], + outputEnumValues: outputEnumValues as ScorerSpec['outputEnumValues'], + agentAssociation, + }; + } + + private async promptForEngineConfig(engineType: string): Promise<{ promptContent?: string; promptTemplateName?: string }> { + if (engineType !== 'PromptTemplate') return {}; + // No flag exists yet for referencing an existing prompt template by name, so in --json/ + // non-interactive mode we always generate a new default prompt template. + if (this.jsonEnabled()) return {}; + + this.log(); + this.styledHeader('Prompt Template'); + + const promptChoice = await select({ + message: 'Prompt template source', + choices: [ + { name: 'Generate a new default prompt template', value: 'generate' }, + { name: 'Use an existing prompt template', value: 'existing' }, + ], + theme, + }); + + if (promptChoice === 'existing') { + const promptTemplateName = await inquirerInput({ + message: 'Existing prompt template API name', + validate: (d: string): boolean | string => d.length > 0 || 'Name cannot be empty', + theme, + }); + return { promptTemplateName }; + } + + return {}; + } + + private async promptForAgentAssociationDetails( + connection: ReturnType, + engineType: string, + agentApiNameFlag?: string + ): Promise { + let agentAssociation: ScorerSpec['agentAssociation']; + if (agentApiNameFlag) { + agentAssociation = { agentApiName: agentApiNameFlag, isActive: false }; + } else { + const agentsInOrg = await Agent.listRemote(connection); + if (!agentsInOrg.length) { + throw messages.createError('error.noAgentsInOrg'); + } + const agentApiName = await select({ + message: 'Select the agent to associate with this scorer', + choices: agentsInOrg + .filter((a) => !a.IsDeleted) + .sort((a, b) => a.DeveloperName.localeCompare(b.DeveloperName)) + .map((a) => ({ name: a.DeveloperName, value: a.DeveloperName })), + theme, + }); + agentAssociation = { agentApiName, isActive: false }; + } + + const associationInputScope = this.jsonEnabled() + ? 'Session' + : await select({ + message: 'Input scope for this agent association', + choices: SCORER_INPUT_SCOPES.map((s) => ({ name: s, value: s })), + default: 'Session', + theme, + }); + agentAssociation.inputScope = associationInputScope as 'Session' | 'Intent'; + + if (engineType === 'PromptTemplate') { + const isActive = this.jsonEnabled() + ? false + : await confirm({ + message: 'Activate scoring for this agent?', + default: false, + theme, + }); + agentAssociation.isActive = isActive; + + if (isActive) { + const samplingRateStr = await promptForFlag({ + message: 'Sampling rate (0.0 - 1.0)', + promptMessage: 'Sampling rate (0.0 to 1.0, where 1.0 = score every session)', + validate: (d: string): boolean | string => { + const n = parseFloat(d); + if (isNaN(n) || n < 0 || n > 1) return 'Must be between 0.0 and 1.0'; + return true; + }, + default: '1.0', + }); + agentAssociation.samplingRate = parseFloat(samplingRateStr); + } + } + + return agentAssociation; + } + +} diff --git a/src/commands/agent/scorer/edit.ts b/src/commands/agent/scorer/edit.ts new file mode 100644 index 00000000..7048afee --- /dev/null +++ b/src/commands/agent/scorer/edit.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join, resolve } from 'node:path'; +import { readFile, writeFile } from 'node:fs/promises'; +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { + setVersionStatusInScorerXml, + setVersionAssociationActiveInScorerXml, + SCORER_VERSION_STATUSES, + type ScorerVersionStatus, +} from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.edit'); + +export type AgentScorerEditResult = { + path: string; + apiName: string; + contents: string; +}; + +/** + * Edit one version of an already-authored scorer, in place: change its status (`--status`) and/or turn its + * agent association on or off (`--activate` / `--deactivate`). Editing never authors content — a version's + * rubric is immutable once it exists; status and activation are the only fields that change. To add a new + * version (a refined rubric) use `sf agent scorer create --new-version`. + * + * This is a purely local metadata operation (it edits the scorer's XML on disk), so it needs no org connection; + * deploy the definition afterward for the org to reflect the change. + */ +export default class AgentScorerEdit extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly flags = { + 'api-name': Flags.string({ + summary: messages.getMessage('flags.api-name.summary'), + required: true, + }), + // eslint-disable-next-line sf-plugin/flag-min-max-default + version: Flags.integer({ + summary: messages.getMessage('flags.version.summary'), + required: true, + min: 1, + }), + status: Flags.string({ + summary: messages.getMessage('flags.status.summary'), + options: SCORER_VERSION_STATUSES, + }), + activate: Flags.boolean({ + summary: messages.getMessage('flags.activate.summary'), + exclusive: ['deactivate'], + }), + deactivate: Flags.boolean({ + summary: messages.getMessage('flags.deactivate.summary'), + exclusive: ['activate'], + }), + 'output-dir': Flags.directory({ + summary: messages.getMessage('flags.output-dir.summary'), + default: join('force-app', 'main', 'default'), + }), + preview: Flags.boolean({ + summary: messages.getMessage('flags.preview.summary'), + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentScorerEdit); + const apiName = flags['api-name']; + const version = flags.version; + const outputDir = resolve(flags['output-dir']); + const status = flags.status as ScorerVersionStatus | undefined; + // --activate → true, --deactivate → false, neither → leave the association untouched. + const activate = flags.activate ? true : flags.deactivate ? false : undefined; + + if (!status && activate === undefined) { + throw messages.createError('error.noChange'); + } + + const scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${apiName}.aiAgentScorerDefinition-meta.xml`); + + let existingXml: string; + try { + existingXml = await readFile(scorerPath, 'utf8'); + } catch (err) { + // Only a missing file means "not authored yet"; surface any other read failure (EACCES, EISDIR, …) + // as-is so the user isn't wrongly told to `create` a scorer that already exists. + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + throw messages.createError('error.scorerNotFound', [apiName, scorerPath]); + } + throw err; + } + + // Apply every requested change in memory against a single load, then write once, so a combined status + + // activation edit (and its --preview) reflects both changes together. + let contents = existingXml; + const changes: string[] = []; + if (status) { + contents = setVersionStatusInScorerXml(contents, apiName, version, status); + changes.push(`status → ${status}`); + } + if (activate !== undefined) { + contents = setVersionAssociationActiveInScorerXml(contents, apiName, version, activate); + changes.push(activate ? 'agent association activated' : 'agent association deactivated'); + } + + if (flags.preview) { + this.log(`\n--- ${apiName} v${version} (${changes.join(', ')}) — preview ---\n`); + this.log(contents); + return { path: scorerPath, apiName, contents }; + } + + await writeFile(scorerPath, contents); + this.log(`Updated ${apiName} v${version} (${changes.join(', ')}): ${scorerPath}`); + this.log('Deploy the scorer definition for the org to reflect this change.'); + + return { path: scorerPath, apiName, contents }; + } +} diff --git a/src/commands/agent/scorer/run.ts b/src/commands/agent/scorer/run.ts new file mode 100644 index 00000000..dd82d616 --- /dev/null +++ b/src/commands/agent/scorer/run.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { resolve } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable } from '@salesforce/core'; +import { type SessionView, type ScorerResult, runScorer, loadScorerSpec, sessionViewJsonSchema } from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.run'); + +// The JSON Schema for the STDM session object, surfaced in the --data flag's help so authors (and agents) +// can hand-construct a valid session. Derived from the SessionView type in @salesforce/agents. +// +// oclif renders flag descriptions through wrap-ansi, which mangles pretty-printed JSON in two ways: it +// tokenizes each line on the regular space (U+0020), and it trimStart()s every wrapped row. Because U+00A0 +// (no-break space) is still matched by JS's \s, a plain NBSP indent gets trimmed away on any line with more +// than one whitespace-separated token. So we indent with no-break spaces (never split on) AND prefix each +// indented line with a zero-width space (U+200B) — a non-whitespace char that halts trimStart() before it +// can reach the indentation, so the nesting survives the formatter. Both chars render as no ink / blank +// columns, keeping the schema readable. +const NBSP = '\u00A0'; +const ZWSP = '\u200B'; +const prettySessionSchema = JSON.stringify(sessionViewJsonSchema(), null, 2).replace( + /^ +/gm, + (spaces) => `${ZWSP}${NBSP.repeat(spaces.length)}` +); +const DATA_SCHEMA_HELP = `${messages.getMessage('flags.data.description')}\n\n${prettySessionSchema}`; + +export type AgentScorerRunResult = ScorerResult & { + scorerApiName: string; + /** The version number that was actually run (highest Available by default, or the requested --scorer-version). */ + scorerVersion?: number; +}; + +export default class AgentScorerRun extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly envVariablesSection = toHelpSection('ENVIRONMENT VARIABLES', EnvironmentVariable.SF_TARGET_ORG); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + 'api-name': Flags.string({ + summary: messages.getMessage('flags.api-name.summary'), + required: true, + }), + // eslint-disable-next-line sf-plugin/flag-min-max-default + 'scorer-version': Flags.integer({ + summary: messages.getMessage('flags.scorer-version.summary'), + min: 1, + }), + data: Flags.string({ + summary: messages.getMessage('flags.data.summary'), + description: DATA_SCHEMA_HELP, + exactlyOne: ['data', 'file'], + }), + file: Flags.file({ + summary: messages.getMessage('flags.file.summary'), + exists: true, + exactlyOne: ['data', 'file'], + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentScorerRun); + + const apiName = flags['api-name']; + + // Resolve the scorer from local project metadata by API name — the business logic throws a clear error + // if no scorer with this API name is authored in the project. + const directories = this.project!.getUniquePackageDirectories().map((pkgDir) => pkgDir.fullPath); + const spec = await loadScorerSpec({ apiName, directories, scorerVersion: flags['scorer-version'] }); + + const session = this.parseSession(flags.file ? readFileSync(resolve(flags.file), 'utf8') : flags.data!); + + const connection = flags['target-org'].getConnection(flags['api-version']); + const result = await runScorer(spec, session, connection); + + if (!this.jsonEnabled()) { + this.styledHeader(`Scorer: ${spec.apiName}`); + if (spec.scorerVersion) this.log(`Version: ${spec.scorerVersion}`); + this.log(`Outcome: ${result.ok ? 'ok' : 'error'}`); + if (result.output !== undefined) { + this.log(`Output: ${Array.isArray(result.output) ? result.output.join(', ') : String(result.output)}`); + } + if (result.explanation) this.log(`Explanation: ${result.explanation}`); + } + + // A scorer that didn't produce a valid score (engine failure, or a session the platform rejected) comes back + // with ok:false. Surface it as a command failure — non-zero exit, standard SfError envelope in --json — so a + // scripted loop can't mistake a failed evaluation for a passing one. The full result is attached as error data. + if (!result.ok) { + const error = messages.createError('error.scorerRunFailed', [spec.apiName, result.error ?? 'unknown error']); + error.data = { scorerApiName: spec.apiName, scorerVersion: spec.scorerVersion, ...result }; + throw error; + } + + return { scorerApiName: spec.apiName, scorerVersion: spec.scorerVersion, ...result }; + } + + // eslint-disable-next-line class-methods-use-this + private parseSession(raw: string): SessionView { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + throw messages.createError('error.invalidSessionJson', [(e as Error).message]); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw messages.createError('error.invalidSessionShape', [typeof parsed]); + } + return parsed as SessionView; + } +} diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts new file mode 100644 index 00000000..d4597bd0 --- /dev/null +++ b/test/commands/agent/scorer/create.test.ts @@ -0,0 +1,1104 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any */ + +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createRequire } from 'node:module'; +import { mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; + +// The command reads specs via node:fs, but createScorerDefinition (in @salesforce/agents) writes +// output via node:fs/promises (writeFile/mkdir). node: core modules are singletons, so stubbing +// this shared instance captures the library's real writes without touching disk. Obtained through +// createRequire because ESM namespace objects are frozen and cannot be stubbed. +const fsPromises = createRequire(import.meta.url)('node:fs/promises') as { + writeFile: (...args: any[]) => Promise; + mkdir: (...args: any[]) => Promise; + readFile: (...args: any[]) => Promise; +}; +import YAML from 'yaml'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import * as agentsModule from '@salesforce/agents'; +import type { ScorerSpec } from '@salesforce/agents'; + +function makeLabeledSpec(overrides: Partial = {}): ScorerSpec { + return { + apiName: 'Test_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Test Scorer', + description: 'A test scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: false, + }, + outputEnumValues: [ + { value: 'Positive', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Negative', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'Neutral', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + ...overrides, + }; +} + +function makeOpenSpec(overrides: Partial = {}): ScorerSpec { + return { + apiName: 'Open_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Open Scorer', + engineType: 'PromptTemplate', + status: 'Draft', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: true, + samplingRate: 0.5, + inputScope: 'Intent', + }, + ...overrides, + }; +} + +function makePromptTemplateSpec(overrides: Partial = {}): ScorerSpec { + return { + apiName: 'Prompt_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Prompt Scorer', + engineType: 'PromptTemplate', + status: 'Draft', + promptContent: 'Evaluate this session.\n\n{!$Input:Session}', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: true, + samplingRate: 1.0, + }, + outputEnumValues: [ + { value: 'Pass', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Fail', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + ...overrides, + }; +} + +type WrittenFile = { path: string; content: string }; + +async function loadMockedCommand( + yamlSpec: ScorerSpec, + opts?: { + existsSync?: () => boolean; + confirmResult?: boolean; + existingScorerXml?: string; + existingTemplateXml?: string; + } +): Promise<{ Command: any; writtenFiles: WrittenFile[]; createdDirs: string[] }> { + const yamlContent = YAML.stringify(yamlSpec); + const writtenFiles: WrittenFile[] = []; + const createdDirs: string[] = []; + const fileExists = opts?.existsSync ?? (() => false); + + const fsMock: Record = { + readFileSync: () => yamlContent, + existsSync: fileExists, + }; + + const mocks: Record = { 'node:fs': fsMock }; + + // Capture the scorer/prompt-template files written by createScorerDefinition (via + // node:fs/promises) without hitting disk; delegate any unrelated writes to the real fns. + const isScorerOutput = (p: unknown): boolean => + typeof p === 'string' && (p.includes('aiAgentScorerDefinitions') || p.includes('genAiPromptTemplates')); + const origWriteFile = fsPromises.writeFile; + const origMkdir = fsPromises.mkdir; + const origReadFile = fsPromises.readFile; + + // addScorerVersion / setScorerVersionStatus read existing metadata via node:fs/promises.readFile. + // Serve the supplied fixture XML for those paths so the version/transition logic runs without disk. + if (opts?.existingScorerXml !== undefined || opts?.existingTemplateXml !== undefined) { + sinon.stub(fsPromises, 'readFile').callsFake((path: unknown, ...rest: any[]) => { + const p = String(path); + if (p.includes('genAiPromptTemplates') && opts.existingTemplateXml !== undefined) { + return Promise.resolve(opts.existingTemplateXml); + } + if (p.includes('aiAgentScorerDefinitions') && opts.existingScorerXml !== undefined) { + return Promise.resolve(opts.existingScorerXml); + } + return origReadFile(path, ...rest); + }); + } + sinon.stub(fsPromises, 'writeFile').callsFake((path: unknown, content: unknown, options: unknown) => { + if (isScorerOutput(path)) { + writtenFiles.push({ path: String(path), content: String(content) }); + return Promise.resolve(); + } + return origWriteFile(path, content, options); + }); + sinon.stub(fsPromises, 'mkdir').callsFake((path: unknown, options: unknown) => { + if (isScorerOutput(path)) { + createdDirs.push(String(path)); + return Promise.resolve(undefined); + } + return origMkdir(path, options); + }); + + if (opts?.confirmResult !== undefined) { + mocks['@inquirer/prompts'] = { + confirm: sinon.stub().resolves(opts.confirmResult), + select: sinon.stub().resolves('lightning__textType'), + input: sinon.stub().resolves(''), + }; + } + + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', mocks); + return { Command: mod.default, writtenFiles, createdDirs }; +} + +// Bare spec filenames used across the --spec tests. `spec: Flags.file({ exists: true })` makes +// oclif stat the path at parse time (via a CJS require of node:fs/promises deep inside +// @oclif/core, which esmock cannot intercept), so these must physically exist on disk. We create +// them in a temp dir and chdir there; the command's readFileSync is still mocked, so file +// contents are irrelevant — only their existence matters. +const SPEC_FILENAMES = [ + 'test.yaml', + 'test-scorer.yaml', + 'open-scorer.yaml', + 'prompt-scorer.yaml', + 'manual-scorer.yaml', +]; + +describe('agent scorer create', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + let originalCwd: string; + let specDir: string; + let sfCommandStubs: ReturnType; + + before(async function () { + // Warm up esmock to check it can load the module + try { + await esmock('../../../../src/commands/agent/scorer/create.js', { + 'node:fs': { + readFileSync: () => '', + writeFileSync: () => {}, + mkdirSync: () => {}, + existsSync: () => false, + }, + }); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + + originalCwd = process.cwd(); + specDir = mkdtempSync(join(tmpdir(), 'scorer-specs-')); + for (const name of SPEC_FILENAMES) writeFileSync(join(specDir, name), ''); + process.chdir(specDir); + }); + + after(() => { + if (originalCwd) process.chdir(originalCwd); + if (specDir) rmSync(specDir, { recursive: true, force: true }); + }); + + beforeEach(async () => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + describe('--spec flag (YAML-driven) with --preview', () => { + it('should create a labeled scorer from a YAML spec', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('AiAgentScorerDefinition'); + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__textType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('Session'); + expect(result.contents).to.include('Manual'); + expect(result.contents).to.include('Draft'); + expect(result.contents).to.include('My_Agent'); + expect(result.contents).to.include('Positive'); + expect(result.contents).to.include('Negative'); + expect(result.contents).to.include('Neutral'); + }); + + it('should create an OpenEnded (LightningType) scorer', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Open_Scorer'); + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__textType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('Session'); + }); + + it('should include inputScope in agent association when specified', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + const agentAssocBlock = result.contents.substring( + result.contents.indexOf(''), + result.contents.indexOf('') + ''.length + ); + expect(agentAssocBlock).to.include('Intent'); + }); + + it('should include outputEnumValues when provided', async () => { + const spec = makeOpenSpec({ + outputEnumValues: [ + { value: 'GOOD', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'BAD', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'N/A', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('GOOD'); + expect(result.contents).to.include('BAD'); + expect(result.contents).to.include('N/A'); + expect(result.contents).to.include('Pass'); + expect(result.contents).to.include('Fail'); + expect(result.contents).to.include('NotApplicable'); + expect(result.contents).to.include('true'); + }); + + it('should not include outputEnumValue when none provided', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).not.to.include(''); + expect(result.contents).not.to.include(''); + }); + + it('should generate prompt template path for PromptTemplate engine', async () => { + const { Command } = await loadMockedCommand(makePromptTemplateSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'prompt-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.promptTemplatePath).to.be.a('string'); + expect(result.promptTemplatePath).to.include('genAiPromptTemplates'); + expect(result.promptTemplatePath).to.include('Prompt_Scorer.genAiPromptTemplate-meta.xml'); + expect(result.contents).to.include('Prompt_Scorer'); + expect(result.contents).to.include('PromptTemplate'); + }); + + it('should not generate prompt template for Manual engine', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ engineType: 'Manual' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'manual-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.promptTemplatePath).to.be.undefined; + expect(result.contents).not.to.include(''); + expect(result.contents).to.include('Manual'); + }); + + it('should use promptTemplateName as engineRef and skip prompt template file generation', async () => { + const spec = makePromptTemplateSpec({ promptTemplateName: 'My_Existing_Template' }); + const { Command, writtenFiles } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.contents).to.include('My_Existing_Template'); + expect(result.contents).to.include('PromptTemplate'); + expect(result.promptTemplatePath).to.be.undefined; + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile).to.be.undefined; + expect(writtenFiles).to.have.length(1); + }); + + it('should omit inputScope from agent association XML when not specified', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.inputScope = undefined; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + const agentAssocBlock = result.contents.substring( + result.contents.indexOf(''), + result.contents.indexOf('') + ''.length + ); + expect(agentAssocBlock).to.include('My_Agent'); + expect(agentAssocBlock).not.to.include(''); + }); + + it('should include description when provided', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ description: 'Evaluates politeness' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Evaluates politeness'); + }); + + it('should omit description when not provided', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ description: undefined })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).not.to.include(''); + }); + + it('should default samplingRate to 1.0', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.samplingRate = undefined; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('1'); + }); + + it('should use custom samplingRate', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.samplingRate = 0.25; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('0.25'); + }); + + it('should set versionNumber to 1', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('1'); + }); + }); + + describe('prompt template type', () => { + it('should always use scorerOpenEnded type', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); + }); + + it('should use scorerOpenEnded type even when labels are defined', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); + expect(promptFile!.content).to.include('AllowedLabels'); + expect(promptFile!.content).to.include('FallbackLabel'); + }); + }); + + describe('XML structure', () => { + it('should include XML declaration and namespace', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include(''); + expect(result.contents).to.include('xmlns="http://soap.sforce.com/2006/04/metadata"'); + }); + + it('should include isActive in agent association', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.isActive = true; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('true'); + }); + + it('should include isFallback and isSystemFallback', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('false'); + expect(result.contents).to.include('true'); + expect(result.contents).to.include('false'); + }); + + it('should include label in scorerVersion', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ label: 'My Custom Label' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include(''); + }); + }); + + describe('file writing', () => { + it('should write scorer XML to correct path', async () => { + const { Command, writtenFiles, createdDirs } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.path).to.include('/tmp/out'); + expect(result.path).to.include('aiAgentScorerDefinitions'); + expect(result.path).to.include('Test_Scorer.aiAgentScorerDefinition-meta.xml'); + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('AiAgentScorerDefinition'); + expect(createdDirs.some((d) => d.includes('aiAgentScorerDefinitions'))).to.be.true; + }); + + it('should write both scorer and prompt template for PromptTemplate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(2); + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(scorerFile).to.not.be.undefined; + expect(promptFile).to.not.be.undefined; + expect(promptFile!.path).to.include('Prompt_Scorer.genAiPromptTemplate-meta.xml'); + expect(promptFile!.content).to.include('GenAiPromptTemplate'); + expect(result.promptTemplatePath).to.equal(promptFile!.path); + }); + + it('should not write files with --preview', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(writtenFiles).to.have.length(0); + }); + + it('should use default prompt content when promptContent not in spec', async () => { + const spec = makePromptTemplateSpec(); + delete (spec as any).promptContent; + const { Command, writtenFiles } = await loadMockedCommand(spec); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).to.include('{!$Input:AllowedLabels}'); + expect(promptFile!.content).to.include('{!$Input:FallbackLabel}'); + }); + + it('should omit label guidance from default prompt when no labels are defined', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); + }); + }); + + describe('output directory', () => { + it('should default to force-app/main/default', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.path).to.include('force-app/main/default/aiAgentScorerDefinitions'); + }); + + it('should use custom --output-dir', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/custom/path', + '--preview', + '--json', + ]); + + expect(result.path).to.include('/custom/path/aiAgentScorerDefinitions'); + }); + }); + + describe('existing scorer behavior', () => { + it('errors when the scorer already exists and --new-version is not passed', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { + existsSync: () => true, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('already exists'); + expect((err as Error).message).to.include('--new-version'); + } + expect(writtenFiles).to.have.length(0); + }); + + it('adds a new version when the scorer exists and --new-version is passed', async () => { + const spec = makeLabeledSpec(); + const existingScorerXml = (agentsModule as any).buildScorerXml(spec); + const { Command, writtenFiles } = await loadMockedCommand(spec, { + existsSync: () => true, + existingScorerXml, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--new-version', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + // v1 is preserved and v2 is appended. + expect(result.contents).to.include('1'); + expect(result.contents).to.include('2'); + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + expect(scorerFile).to.not.be.undefined; + }); + + it('previews the appended version (not a fresh v1) with --new-version --preview, writing nothing', async () => { + const spec = makeLabeledSpec(); + const existingScorerXml = (agentsModule as any).buildScorerXml(spec); + const { Command, writtenFiles } = await loadMockedCommand(spec, { + existsSync: () => true, + existingScorerXml, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--new-version', + '--preview', + '--json', + ]); + + // Preview reflects the artifact --new-version would write: v1 preserved, v2 appended — not a fresh v1. + expect(result.contents).to.include('1'); + expect(result.contents).to.include('2'); + expect(writtenFiles).to.have.length(0); + }); + }); + + describe('prompt template XML details', () => { + it('should include developerName and masterLabel matching apiName', async () => { + const { Command, writtenFiles } = await loadMockedCommand( + makePromptTemplateSpec({ apiName: 'My_Prompt_Scorer' }) + ); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('My_Prompt_Scorer'); + expect(promptFile!.content).to.include('My_Prompt_Scorer'); + }); + + it('should set overridable to false and visibility to Global', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('false'); + expect(promptFile!.content).to.include('Global'); + }); + + it('should set primaryModel and status Published', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('sfdc_ai__DefaultOpenAIGPT4OmniMini'); + expect(promptFile!.content).to.include('Published'); + }); + + it('should include activeVersionIdentifier and versionIdentifier', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include(''); + expect(promptFile!.content).to.include(''); + // Both should have the same value + const activeMatch = promptFile!.content.match(/(.+?)<\/activeVersionIdentifier>/); + const versionMatch = promptFile!.content.match(/(.+?)<\/versionIdentifier>/); + expect(activeMatch![1]).to.equal(versionMatch![1]); + expect(activeMatch![1]).to.match(/.+=_1$/); + }); + + it('should include Session input with correct definition', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include( + 'lightningtype://propertyType/agentforce_session_tracing__stdmDetailViewType' + ); + expect(promptFile!.content).to.include('Input:Session'); + }); + }); + + describe('--json mode error handling', () => { + it('should throw when required flags are missing', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + try { + await Command.run(['--target-org', testOrg.username, '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('Missing required flags'); + } + }); + + it('should list all missing required flags', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + try { + await Command.run(['--target-org', testOrg.username, '--label', 'Foo', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('api-name'); + expect(error.message).to.include('lightning-type'); + expect(error.message).to.include('engine-type'); + expect(error.message).to.include('agent-api-name'); + } + }); + }); + + describe('output label validation', () => { + let tmpDir: string; + let specFile: string; + + beforeEach(() => { + tmpDir = join(process.cwd(), 'tmp-test-fallback-' + Date.now()); + mkdirSync(tmpDir, { recursive: true }); + specFile = join(tmpDir, 'scorer.yaml'); + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('should throw when more than one output value is the fallback', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: true, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('At most one outputEnumValue can be the fallback'); + expect(error.message).to.include('found 2'); + } + }); + + it('should pass with zero fallback values', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('Good'); + }); + + it('should pass with exactly one fallback value', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'N/A', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('N/A'); + }); + }); + + describe('edge cases', () => { + it('should handle LightningType with no outputEnumValues', async () => { + const spec: ScorerSpec = { + apiName: 'Lightning_Scorer', + lightningType: 'lightning__numberType', + inputScope: 'Session', + label: 'Lightning Scorer', + engineType: 'Manual', + agentAssociation: { agentApiName: 'Agent_X', isActive: false }, + }; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__numberType'); + }); + + it('should handle single output enum value', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Only', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Only'); + expect(result.contents).to.include('NotApplicable'); + expect(result.contents).to.include('true'); + }); + }); + + describe('--json without --spec (flag-only path)', () => { + it('completes without prompting when all required flags are supplied', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--label', 'My Scorer', + '--api-name', 'My_Scorer', + '--lightning-type', 'lightning__textType', + '--engine-type', 'Manual', + '--agent-api-name', 'My_Agent', + '--output-dir', '/tmp/out-json-flags', + '--json', + ]); + + expect(result.apiName).to.equal('My_Scorer'); + // The status flag's default ('Draft') must be honored without prompting. + expect(result.contents).to.include('Draft'); + expect(writtenFiles).to.have.length(1); + }); + }); + + describe('--spec YAML parsing', () => { + it('throws a clear error when the spec file is not valid YAML', async () => { + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', { + 'node:fs': { readFileSync: () => 'foo: [1, 2', existsSync: () => false }, + }); + const Command = mod.default; + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('Could not parse the --spec file as YAML'); + } + }); + + it('throws a clear error when the spec file is not a YAML object', async () => { + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', { + 'node:fs': { readFileSync: () => '- 1\n- 2\n', existsSync: () => false }, + }); + const Command = mod.default; + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('must define a YAML object'); + } + }); + }); + + describe('--spec-schema', () => { + it('prints the schema JSON and skips org/spec resolution', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + // Deliberately omit --spec and every other required flag: --spec-schema must short-circuit + // before the required-flags gate or spec/org resolution ever runs. + const result = await Command.run(['--target-org', testOrg.username, '--spec-schema']); + + expect(result).to.deep.equal({ path: '', apiName: '', contents: '' }); + expect(sfCommandStubs.styledJSON.calledOnce).to.be.true; + const printed = sfCommandStubs.styledJSON.firstCall.args[0] as Record; + expect(printed).to.have.property('$ref', '#/definitions/ScorerSpec'); + expect(printed).to.have.property('definitions'); + }); + }); + + describe('interactive interview (no --spec, no --json)', () => { + it('throws a clear error when the org has no agents to associate', async () => { + const mocks: Record = { + 'node:fs': { readFileSync: () => '', existsSync: () => false }, + '@inquirer/prompts': { + input: sinon.stub().resolves(''), + confirm: sinon.stub().resolves(false), + select: sinon.stub().resolves(''), + }, + '@salesforce/agents': { + ...agentsModule, + Agent: { listRemote: sinon.stub().resolves([]) }, + }, + }; + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', mocks); + const Command = mod.default; + + try { + // Supply every FLAGGABLE_PROMPTS-backed flag (including --description and --status, which + // are otherwise resolved via promptForFlag() in ../../../flags.js — a module esmock does not + // remock here, so any prompt routed through it would hit the real @inquirer/prompts and hang). + await Command.run([ + '--target-org', testOrg.username, + '--label', 'My Scorer', + '--api-name', 'My_Scorer', + '--lightning-type', 'lightning__textType', + '--engine-type', 'Manual', + '--description', 'A test description', + '--status', 'Draft', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('No agents found in the org'); + } + }); + }); +}); diff --git a/test/commands/agent/scorer/edit.test.ts b/test/commands/agent/scorer/edit.test.ts new file mode 100644 index 00000000..432707ee --- /dev/null +++ b/test/commands/agent/scorer/edit.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any */ + +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; +import { TestContext } from '@salesforce/core/testSetup'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import * as agentsModule from '@salesforce/agents'; + +type WrittenFile = { path: string; content: string }; + +const SPEC: any = { + apiName: 'Test_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Test Scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, +}; + +const SPEC_ACTIVE: any = { ...SPEC, agentAssociation: { agentApiName: 'My_Agent', isActive: true } }; + +/** Build a scorer XML fixture with `versions` sequential versions (v1..vN) from `spec`. */ +function scorerXmlWithVersions(versions: number, spec: any = SPEC): string { + let xml = (agentsModule as any).buildScorerXml(spec); + for (let i = 2; i <= versions; i++) { + ({ xml } = (agentsModule as any).addVersionToScorerXml(xml, spec)); + } + return String(xml); +} + +// The edit command reads/writes the scorer XML via node:fs/promises. esmock swaps that module for the command +// only, so reads return a fixture and writes are captured — no disk I/O and no reliance on core-module stubbing. +async function loadMockedCommand( + existingScorerXml: string | null, + readError?: NodeJS.ErrnoException +): Promise<{ Command: any; writtenFiles: WrittenFile[] }> { + const writtenFiles: WrittenFile[] = []; + + const readFile = (): Promise => { + // readError models a non-ENOENT read failure (EACCES, EISDIR, …); null models a missing file (ENOENT). + if (readError) return Promise.reject(readError); + return existingScorerXml == null + ? Promise.reject(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })) + : Promise.resolve(existingScorerXml); + }; + + const writeFile = (path: unknown, content: unknown): Promise => { + writtenFiles.push({ path: String(path), content: String(content) }); + return Promise.resolve(); + }; + + const mod = await esmock('../../../../src/commands/agent/scorer/edit.js', { + 'node:fs/promises': { readFile, writeFile }, + }); + return { Command: mod.default, writtenFiles }; +} + +describe('agent scorer edit', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + + before(async function () { + try { + await esmock('../../../../src/commands/agent/scorer/edit.js', {}); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + }); + + beforeEach(() => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + it('promotes a version to Available with --status Available', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + const result = await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('Available'); + }); + + it('archives a version with --status Archived', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Archived', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('Archived'); + }); + + it('activates the agent association with --activate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--activate', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('true'); + }); + + it('deactivates the agent association with --deactivate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1, SPEC_ACTIVE)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--deactivate', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('false'); + }); + + it('changes status and activation together in a single write', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--activate', + '--output-dir', '/tmp/out', + '--json', + ]); + + // A single load → both mutations → one write, so the written XML carries both changes. + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('Available'); + expect(writtenFiles[0].content).to.include('true'); + }); + + it('errors when neither --status nor --activate/--deactivate is provided', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + try { + await Command.run(['--api-name', 'Test_Scorer', '--version', '1', '--output-dir', '/tmp/out', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/status|activate|deactivate/); + } + expect(writtenFiles).to.have.length(0); + }); + + it('rejects --activate together with --deactivate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + try { + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--activate', + '--deactivate', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/activate|deactivate/); + } + expect(writtenFiles).to.have.length(0); + }); + + it('errors when the scorer file is not found', async () => { + const { Command, writtenFiles } = await loadMockedCommand(null); + + try { + await Command.run([ + '--api-name', 'Missing_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('was found at'); + } + expect(writtenFiles).to.have.length(0); + }); + + it('rethrows a non-ENOENT read error instead of reporting "scorer not found"', async () => { + const eacces = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }); + const { Command, writtenFiles } = await loadMockedCommand('unused', eacces); + + try { + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + // the raw fs error propagates; the user is NOT wrongly told the scorer was not found + expect((err as Error).message).to.include('EACCES'); + expect((err as Error).message).to.not.include('was found at'); + } + expect(writtenFiles).to.have.length(0); + }); + + it('writes nothing with --preview but prints the resulting XML', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + const result = await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--preview', + ]); + + expect(writtenFiles).to.have.length(0); + expect(result.contents).to.include('Available'); + const logged = sfCommandStubs.log.args.map((a) => String(a[0])).join('\n'); + expect(logged).to.include('Available'); + }); +}); diff --git a/test/commands/agent/scorer/run.test.ts b/test/commands/agent/scorer/run.test.ts new file mode 100644 index 00000000..86db38ca --- /dev/null +++ b/test/commands/agent/scorer/run.test.ts @@ -0,0 +1,471 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ + +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; + +const SPEC = { + apiName: 'Sentiment_Scorer', + lightningType: 'lightning__textType', + label: 'Sentiment Scorer', + engineType: 'PromptTemplate', + agentAssociation: { agentApiName: 'My_Agent', isActive: true }, +}; + +const SESSION = { + sessionState: { sessionId: 'S1', startTimestamp: '2026-01-01T00:00:00Z', channel: 'web' }, + actors: [], + metrics: { durationMs: 0, turns: 0 }, + runs: [], +}; + +const SESSION_FILE = 'session.json'; + +async function loadMockedCommand(opts?: { + runScorerResult?: any; + runScorerError?: Error; + loadScorerSpecError?: Error; + specOverride?: any; + schema?: unknown; +}): Promise<{ Command: any; runScorer: sinon.SinonStub; loadScorerSpec: sinon.SinonStub }> { + const runScorer = sinon.stub(); + if (opts?.runScorerError) runScorer.rejects(opts.runScorerError); + else runScorer.resolves(opts?.runScorerResult ?? { ok: true, output: 'Positive', explanation: 'Looks good.' }); + + const loadScorerSpec = sinon.stub(); + if (opts?.loadScorerSpecError) loadScorerSpec.rejects(opts.loadScorerSpecError); + else loadScorerSpec.resolves(opts?.specOverride ?? SPEC); + + const readFileSync = (path: unknown): string => { + const p = String(path); + if (p.endsWith('.json')) return JSON.stringify(SESSION); + return ''; + }; + + const mocks: Record = { + 'node:fs': { readFileSync }, + '@salesforce/agents': { + runScorer, + loadScorerSpec, + sessionViewJsonSchema: () => opts?.schema ?? { $schema: 'http://json-schema.org/draft-07/schema#' }, + }, + }; + + const mod = await esmock('../../../../src/commands/agent/scorer/run.js', mocks); + return { Command: mod.default, runScorer, loadScorerSpec }; +} + +describe('agent scorer run', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + let originalCwd: string; + let workDir: string; + let sfCommandStubs: ReturnType; + + before(async function () { + try { + await esmock('../../../../src/commands/agent/scorer/run.js', { + 'node:fs': { readFileSync: () => '' }, + '@salesforce/agents': { + runScorer: () => Promise.resolve({ ok: true }), + loadScorerSpec: () => Promise.resolve(SPEC), + sessionViewJsonSchema: () => ({}), + }, + }); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + + originalCwd = process.cwd(); + workDir = mkdtempSync(join(tmpdir(), 'scorer-run-')); + // Flags.file({ exists: true }) stats this at parse time, so it must exist on disk. + writeFileSync(join(workDir, SESSION_FILE), ''); + process.chdir(workDir); + }); + + after(() => { + if (originalCwd) process.chdir(originalCwd); + if (workDir) rmSync(workDir, { recursive: true, force: true }); + }); + + beforeEach(async () => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + + // requiresProject: stub the project so package directories resolve without a real sfdx-project.json. + $$.inProject(true); + const mockProject = { + getPath: () => workDir, + getUniquePackageDirectories: () => [{ fullPath: join(workDir, 'force-app') }], + } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + it('resolves the scorer by API name and runs it against a session file', async () => { + const { Command, runScorer, loadScorerSpec } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerApiName).to.equal('Sentiment_Scorer'); + expect(result.ok).to.equal(true); + expect(result.output).to.equal('Positive'); + expect(result.explanation).to.equal('Looks good.'); + + expect(loadScorerSpec.calledOnce).to.be.true; + expect(loadScorerSpec.firstCall.firstArg.apiName).to.equal('Sentiment_Scorer'); + expect(loadScorerSpec.firstCall.firstArg.directories).to.be.an('array').that.is.not.empty; + + expect(runScorer.calledOnce).to.be.true; + const [passedSpec, passedSession] = runScorer.firstCall.args; + expect(passedSpec.apiName).to.equal('Sentiment_Scorer'); + expect(passedSession.sessionState.sessionId).to.equal('S1'); + }); + + it('plumbs --scorer-version through to loadScorerSpec', async () => { + const { Command, loadScorerSpec } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--scorer-version', '3', + '--file', SESSION_FILE, + '--json', + ]); + + expect(loadScorerSpec.calledOnce).to.be.true; + expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.equal(3); + }); + + it('leaves scorerVersion undefined when --scorer-version is omitted', async () => { + const { Command, loadScorerSpec } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.be.undefined; + }); + + describe('resolved-version reporting', () => { + it('reports the resolved scorerVersion in the JSON result', async () => { + const { Command } = await loadMockedCommand({ specOverride: { ...SPEC, scorerVersion: 2 } }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerVersion).to.equal(2); + }); + + it('prints the resolved version in human-readable output', async () => { + const { Command } = await loadMockedCommand({ specOverride: { ...SPEC, scorerVersion: 2 } }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Version: 2'); + }); + + it('omits scorerVersion (and the Version line) when the spec has no resolved version', async () => { + const { Command } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerVersion).to.be.undefined; + }); + + it('includes the resolved scorerVersion in error.data on a failed result', async () => { + const { Command } = await loadMockedCommand({ + specOverride: { ...SPEC, scorerVersion: 2 }, + runScorerResult: { ok: false, error: 'no engine for Manual' }, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { data?: any }; + expect(error.data?.scorerVersion).to.equal(2); + } + }); + }); + + it('runs a scorer against inline session JSON', async () => { + const { Command, runScorer } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', JSON.stringify(SESSION), + '--json', + ]); + + expect(result.scorerApiName).to.equal('Sentiment_Scorer'); + expect(result.ok).to.equal(true); + const [, passedSession] = runScorer.firstCall.args; + expect(passedSession.sessionState.sessionId).to.equal('S1'); + }); + + it('passes the connection from the target org to runScorer', async () => { + const { Command, runScorer } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + const [, , connection] = runScorer.firstCall.args; + expect(connection).to.not.be.undefined; + }); + + it('exits non-zero (throws) on an engine error result, with the result attached as error data', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: false, error: 'no engine for Manual' }, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string; data?: any }; + expect(error.message).to.include('did not produce a valid score'); + expect(error.message).to.include('no engine for Manual'); + // The full result is preserved for scripted consumers on the error's data field. + expect(error.data?.ok).to.equal(false); + expect(error.data?.scorerApiName).to.equal('Sentiment_Scorer'); + } + }); + + it('surfaces the error when the scorer is not found in the project', async () => { + const { Command } = await loadMockedCommand({ + loadScorerSpecError: new Error("No scorer named 'Missing_Scorer' was found in this project."), + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Missing_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('was found in this project'); + } + }); + + it('throws when neither --data nor --file is provided', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message.toLowerCase()).to.match(/data|file/); + } + }); + + it('throws when both --data and --file are provided', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', JSON.stringify(SESSION), + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message.toLowerCase()).to.match(/data|file/); + } + }); + + it('throws a clear error when the session is not valid JSON', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', '{not valid json', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('Could not parse the STDM session as JSON'); + } + }); + + it('throws a clear error when the session JSON is not an object', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', '["not", "an", "object"]', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('must be a JSON object'); + } + }); + + describe('human-readable output (non --json)', () => { + it('renders an array output joined by commas', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: ['A', 'B'] }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Output: A, B'); + }); + + it('renders a numeric output', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: 42 }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Output: 42'); + }); + + it('renders the outcome/output/explanation lines before throwing on a failed result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: false, output: 'Negative', explanation: 'Tone was hostile.', error: 'no engine for Manual' }, + }); + + try { + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('no engine for Manual'); + } + + // The human-readable summary is still printed before the command fails. + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Outcome: error'); + expect(logLines).to.include('Output: Negative'); + expect(logLines).to.include('Explanation: Tone was hostile.'); + }); + }); + + describe('--json with array/number output', () => { + it('returns array output as-is in the JSON result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: ['A', 'B'] }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.output).to.deep.equal(['A', 'B']); + }); + + it('returns numeric output as-is in the JSON result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: 7 }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.output).to.equal(7); + }); + }); + + describe('--data schema help formatting', () => { + it('preserves JSON nesting through the ZWSP/NBSP indent markers', async () => { + const schema = { type: 'object', properties: { foo: { type: 'string' }, bar: { type: 'number' } } }; + const { Command } = await loadMockedCommand({ schema }); + + const description = Command.flags.data.description as string; + const separatorIndex = description.indexOf('\n\n'); + expect(separatorIndex).to.be.greaterThan(-1); + const jsonPart = description.slice(separatorIndex + 2); + + // Strip the zero-width-space + no-break-space indent markers (see run.ts) before parsing. + const stripped = jsonPart.replace(/\u200B/g, '').replace(/\u00A0/g, ' '); + const parsed = JSON.parse(stripped); + expect(parsed).to.deep.equal(schema); + }); + }); +}); diff --git a/test/nuts/agent.scorer.create.nut.ts b/test/nuts/agent.scorer.create.nut.ts new file mode 100644 index 00000000..07b4a43e --- /dev/null +++ b/test/nuts/agent.scorer.create.nut.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join } from 'node:path'; +import { writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { expect } from 'chai'; +import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { parseScorerVersions } from '@salesforce/agents'; +import type { AgentScorerCreateResult } from '../../src/commands/agent/scorer/create.js'; + +// `agent scorer create` writes local metadata only, but its `--target-org` is a required flag that resolves at +// parse time, so this NUT needs a default org. It uses a lightweight scratch org (devhub only — no Einstein +// provisioning or metadata deploy, which create doesn't need) rather than the shared heavyweight session. +describe('agent scorer create NUTs', function () { + this.timeout(15 * 60 * 1000); + + const API_NAME = 'Nut_Create_Scorer'; + // A `Manual`-engine scorer needs no prompt template, so create writes a single self-contained definition file. + const makeSpec = (apiName: string, label: string): Record => ({ + apiName, + lightningType: 'lightning__textType', + inputScope: 'Session', + label, + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, + }); + + let session: TestSession; + let outputDir: string; + let specPath: string; + let scorerPath: string; + + before(async () => { + session = await TestSession.create({ + project: { name: 'scorerCreateNut' }, + devhubAuthStrategy: 'AUTO', + scratchOrgs: [{ setDefault: true, config: join('config', 'project-scratch-def.json') }], + }); + outputDir = join(session.project.dir, 'scorer-out'); + specPath = join(session.project.dir, 'nut-create-spec.json'); + scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${API_NAME}.aiAgentScorerDefinition-meta.xml`); + writeFileSync(specPath, JSON.stringify(makeSpec(API_NAME, 'NUT Create Scorer'))); + }); + + after(async () => { + await session?.clean(); + }); + + it('prints the spec JSON Schema with --spec-schema', () => { + const { stdout } = execCmd('agent scorer create --spec-schema', { ensureExitCode: 0 }).shellOutput; + expect(stdout).to.include('ScorerSpec'); + expect(stdout).to.include('apiName'); + }); + + it('authors a scorer definition from a --spec file', () => { + const result = execCmd( + `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.apiName).to.equal(API_NAME); + expect(result?.path).to.equal(scorerPath); + expect(existsSync(scorerPath)).to.equal(true); + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(1); + }); + + it('appends a new version with --new-version', () => { + execCmd( + `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --new-version --json`, + { ensureExitCode: 0 } + ); + + const versions = parseScorerVersions(readFileSync(scorerPath, 'utf8')); + expect(versions.map((v) => v.versionNumber)).to.deep.equal([1, 2]); + }); + + it('refuses to overwrite an existing scorer without --new-version', () => { + const output = execCmd( + `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(new RegExp(API_NAME)); + // the existing file is left untouched (still two versions from the prior test) + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(2); + }); + + it('writes nothing with --preview', () => { + const previewName = 'Nut_Preview_Scorer'; + const previewSpec = join(session.project.dir, 'nut-preview-spec.json'); + writeFileSync(previewSpec, JSON.stringify(makeSpec(previewName, 'NUT Preview Scorer'))); + const previewPath = join(outputDir, 'aiAgentScorerDefinitions', `${previewName}.aiAgentScorerDefinition-meta.xml`); + + const result = execCmd( + `agent scorer create --spec "${previewSpec}" --output-dir "${outputDir}" --preview --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.contents).to.include('AiAgentScorerDefinition'); + expect(existsSync(previewPath)).to.equal(false); + }); +}); diff --git a/test/nuts/agent.scorer.edit.nut.ts b/test/nuts/agent.scorer.edit.nut.ts new file mode 100644 index 00000000..06af70dd --- /dev/null +++ b/test/nuts/agent.scorer.edit.nut.ts @@ -0,0 +1,167 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join } from 'node:path'; +import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'; +import { expect } from 'chai'; +import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { buildScorerXml, addVersionToScorerXml, parseScorerVersions, type ScorerSpec } from '@salesforce/agents'; +import type { AgentScorerEditResult } from '../../src/commands/agent/scorer/edit.js'; + +// `agent scorer edit` is a purely local XML edit — it needs NO org connection — so this NUT runs against a bare +// project with no scratch org. It authors a scorer fixture on disk (via the agents lib, the same way `create` +// would), then exercises the real command end to end: status changes, activation toggles, and the error paths. +describe('agent scorer edit NUTs', () => { + const API_NAME = 'Nut_Edit_Scorer'; + // A `Manual`-engine scorer needs no prompt template, keeping the fixture self-contained. + const spec: ScorerSpec = { + apiName: API_NAME, + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'NUT Edit Scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, + }; + + let session: TestSession; + let outputDir: string; + let scorerPath: string; + + before(async () => { + session = await TestSession.create({ project: { name: 'scorerEditNut' } }); + outputDir = join(session.project.dir, 'force-app', 'main', 'default'); + scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${API_NAME}.aiAgentScorerDefinition-meta.xml`); + }); + + after(async () => { + await session?.clean(); + }); + + /** (Re)author a fresh two-version fixture (v1, v2 — both Draft, inactive) so each test starts from a known state. */ + function authorFixture(active = false): void { + const seed = active + ? { ...spec, agentAssociation: { agentApiName: 'My_Agent', isActive: true } } + : spec; + let xml = buildScorerXml(seed); + ({ xml } = addVersionToScorerXml(xml, seed)); + mkdirSync(join(outputDir, 'aiAgentScorerDefinitions'), { recursive: true }); + writeFileSync(scorerPath, xml); + } + + const versionsOnDisk = (): ReturnType => parseScorerVersions(readFileSync(scorerPath, 'utf8')); + + beforeEach(() => authorFixture()); + + it('promotes a version to Available with --status Available', () => { + const result = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.apiName).to.equal(API_NAME); + const versions = versionsOnDisk(); + expect(versions.find((v) => v.versionNumber === 2)?.status).to.equal('Available'); + // untouched versions keep their status + expect(versions.find((v) => v.versionNumber === 1)?.status).to.equal('Draft'); + }); + + it('archives a version with --status Archived', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 1 --status Archived --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + expect(versionsOnDisk().find((v) => v.versionNumber === 1)?.status).to.equal('Archived'); + }); + + it('activates the agent association with --activate', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --activate --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.isActive).to.equal(true); + }); + + it('deactivates the agent association with --deactivate', () => { + authorFixture(true); + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --deactivate --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.isActive).to.equal(false); + }); + + it('changes status and activation together in a single invocation', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --activate --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + const v2 = versionsOnDisk().find((v) => v.versionNumber === 2); + expect(v2?.status).to.equal('Available'); + expect(v2?.isActive).to.equal(true); + }); + + it('writes nothing with --preview but prints the resulting XML', () => { + const before = readFileSync(scorerPath, 'utf8'); + const output = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --output-dir "${outputDir}" --preview`, + { ensureExitCode: 0 } + ); + + // file untouched... + expect(readFileSync(scorerPath, 'utf8')).to.equal(before); + expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.status).to.equal('Draft'); + // ...but the promoted XML was printed + expect(output.shellOutput.stdout).to.include('Available'); + }); + + it('errors when neither --status nor --activate/--deactivate is provided', () => { + const output = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 1 --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(/status|activate|deactivate/); + }); + + it('rejects --activate together with --deactivate', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 1 --activate --deactivate --output-dir "${outputDir}" --json`, + { ensureExitCode: 'nonZero' } + ); + }); + + it('errors on an unknown version', () => { + const output = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 9 --status Available --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(/version 9/); + }); + + it('errors when the scorer file is not found', () => { + const output = execCmd( + `agent scorer edit --api-name Missing_Scorer --version 1 --status Available --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.include('was found at'); + }); +});