feat: Agentforce Scorers Authoring Command

messages/agent.scorer.create.md

@@ -0,0 +1,107 @@
+# summary
+
+Create an agent scorer definition using an interactive interview or a spec file.
+
+# description
+
+Creates an AiAgentScorerDefinition metadata XML file either interactively (prompting for each field) or from a YAML spec file.
+
+Run with no flags to start the interactive interview. The command prompts you for the scorer's lightning type, optional output labels, engine type, and agent associations.
+
+Alternatively, provide a --spec flag pointing to a YAML file that defines the scorer. This is useful for repeatable automation or when the scorer has many output values.
+
+Use --preview to see the generated XML without writing it to disk.
+
+# flags.api-name.summary
+
+API name of the scorer definition.
+
+# flags.agent-api-name.summary
+
+API name of the agent to associate with this scorer.
+
+# flags.lightning-type.summary
+
+Lightning type the scorer's value conforms to (for example, lightning__textType or lightning__numberType).
+
+# flags.label.summary
+
+Display label for the scorer version.
+
+# flags.description.summary
+
+Description of what this scorer evaluates.
+
+# flags.engine-type.summary
+
+Engine type for scoring (Manual or PromptTemplate).
+
+# flags.status.summary
+
+Initial status of the scorer version (Available or Draft).
+
+# flags.spec.summary
+
+Path to a scorer spec YAML file. Bypasses interactive prompts.
+
+# flags.spec-schema.summary
+
+Output the JSON Schema for the --spec YAML file and exit.
+
+# flags.new-version.summary
+
+Add a new version to an existing scorer instead of erroring. The new version is numbered one higher than the current highest; if a new prompt rubric is supplied, the prompt template's active version is updated too.
+
+# flags.output-dir.summary
+
+Output directory for the generated metadata XML files (scorer definition and prompt template).
+
+# flags.preview.summary
+
+Preview the generated XML without writing to disk.
+
+# examples
+
+- Show the JSON Schema for the spec YAML file:
+
+  <%= config.bin %> <%= command.id %> --spec-schema
+
+- Create a scorer interactively:
+
+  <%= config.bin %> <%= command.id %>
+
+- Create a scorer from a spec file:
+
+  <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml
+
+- Preview the XML that would be generated:
+
+  <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml --preview
+
+- Create a manual scorer with flags (non-interactive):
+
+  <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning__textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available
+
+- Create a prompt-based scorer (generates both scorer definition and prompt template):
+
+  <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning__textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent
+
+# error.missingRequiredFlags
+
+Missing required flags: %s. When using --json, all required flags must be provided.
+
+# error.invalidSpecYaml
+
+Could not parse the --spec file as YAML: %s
+
+# error.invalidSpecShape
+
+The --spec file must define a YAML object matching the scorer spec schema (see --spec-schema). Received: %s
+
+# error.noAgentsInOrg
+
+No agents found in the org. Deploy an agent first, or specify one with --agent-api-name.
+
+# error.scorerExists
+
+A scorer named '%s' already exists in this project. To refine it, add a new version with --new-version; to change a version's status use `sf agent scorer edit`; or use --preview to see the generated XML without writing.

messages/agent.scorer.edit.md

@@ -0,0 +1,69 @@
+# summary
+
+Change the status or agent-association activation of a version of an existing agent scorer.
+
+# description
+
+Edits one version of a scorer in place: promote it (`--status Available`, so its rubric is served by default on the next run), archive it (`--status Archived`, so it can no longer be run), and/or turn its agent association on or off (`--activate` / `--deactivate`). Editing a scorer never authors content: a version's rubric is immutable once it exists — status and activation are the only fields that change. To add a new version with a refined rubric, use `sf agent scorer create --new-version`.
+
+This command edits the scorer's local metadata XML only; it does not require an org connection. Deploy the updated scorer definition afterward for the org to reflect the change.
+
+Platform activation rules (enforced on deploy): an active association (`--activate`) requires the version's status to be `Available`, and at most one version of a scorer may hold an active association.
+
+# flags.api-name.summary
+
+API name of the scorer to edit. Must match a scorer authored in this project's metadata.
+
+# flags.version.summary
+
+Version number to edit.
+
+# flags.status.summary
+
+New status for the version: Draft, Available (promote — served by default on the next run), or Archived (can no longer be run).
+
+# flags.activate.summary
+
+Activate the version's agent association (start scoring the associated agent's sessions). Requires the version's status to be Available.
+
+# flags.deactivate.summary
+
+Deactivate the version's agent association (stop scoring the associated agent's sessions).
+
+# flags.output-dir.summary
+
+Directory containing the scorer's metadata XML (where the scorer definition was authored).
+
+# flags.preview.summary
+
+Preview the resulting XML without writing to disk.
+
+# examples
+
+- Promote version 2 of a scorer to Available:
+
+  <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available
+
+- Archive version 1 so it can no longer be run:
+
+  <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 1 --status Archived
+
+- Promote a version and activate its agent association in a single command:
+
+  <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --activate
+
+- Deactivate the agent association on a version:
+
+  <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --deactivate
+
+- Preview a status change without writing to disk:
+
+  <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --preview
+
+# error.noChange
+
+Specify at least one change: --status, --activate, or --deactivate.
+
+# error.scorerNotFound
+
+No scorer '%s' was found at %s. Author it first with `sf agent scorer create`.

messages/agent.scorer.run.md

@@ -0,0 +1,59 @@
+# summary
+
+Run an agent scorer against an STDM session and print its score.
+
+# description
+
+Runs a scorer that is already authored in your project metadata, referenced by its API name, against a single STDM (Session Trace Data Model) session, then prints the resulting score, outcome labels, and explanation.
+
+The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer create`.
+
+Provide the session either inline as a JSON string with --data, or as a path to a local JSON file with --file. Exactly one of the two is required.
+
+To help you hand-construct a valid session, run this command with --help: the full JSON Schema for the session object is printed under the --data flag.
+
+# flags.api-name.summary
+
+API name of the scorer to run. Must match a scorer authored in this project's metadata.
+
+# flags.scorer-version.summary
+
+Version number of the scorer to run. Omit to run the highest-numbered Available version; the command errors if none is Available. A Draft version must be selected explicitly; an Archived version cannot be run.
+
+# flags.data.summary
+
+Inline STDM session JSON to score.
+
+# flags.data.description
+
+Inline STDM session JSON to score. The value must be a JSON object matching the following JSON Schema (the shape of the scorer's Input:Session value):
+
+# flags.file.summary
+
+Path to a local JSON file containing the STDM session to score.
+
+# examples
+
+- Run a scorer against a session stored in a local file:
+
+  <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --file ./session.json
+
+- Run a scorer against an inline session JSON string:
+
+  <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --data '{"sessionState":{"sessionId":"1","startTimestamp":"2026-01-01T00:00:00Z","channel":"web"},"actors":[],"metrics":{"durationMs":0,"turns":0},"runs":[]}'
+
+- Show the session JSON Schema in the help output:
+
+  <%= config.bin %> <%= command.id %> --help
+
+# error.invalidSessionJson
+
+Could not parse the STDM session as JSON: %s
+
+# error.invalidSessionShape
+
+The STDM session must be a JSON object matching the session schema (see --help). Received: %s
+
+# error.scorerRunFailed
+
+Scorer '%s' did not produce a valid score: %s. The command exits non-zero so a scripted loop won't treat a failed evaluation as a passing one; the full result (including any explanation) is in the error's "data" field when run with --json.

package.json

@@ -95,6 +95,10 @@
             "description": "Command to validate an Agent Script file.",
             "external": true
           },
+          "scorer": {
+            "description": "Commands to create and manage agent scorers.",
+            "external": true
+          },
           "adl": {
             "description": "Commands to manage Agentforce Data Libraries.",
             "external": true,

schemas/agent-scorer-create.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentScorerCreateResult",
+  "definitions": {
+    "AgentScorerCreateResult": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "apiName": {
+          "type": "string"
+        },
+        "contents": {
+          "type": "string"
+        },
+        "promptTemplatePath": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "path",
+        "apiName",
+        "contents"
+      ],
+      "additionalProperties": false
+    }
+  }
+}

schemas/agent-scorer-edit.json

@@ -0,0 +1,26 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentScorerEditResult",
+  "definitions": {
+    "AgentScorerEditResult": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "apiName": {
+          "type": "string"
+        },
+        "contents": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "path",
+        "apiName",
+        "contents"
+      ],
+      "additionalProperties": false
+    }
+  }
+}

schemas/agent-scorer-run.json

@@ -0,0 +1,49 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentScorerRunResult",
+  "definitions": {
+    "AgentScorerRunResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "scorerApiName": {
+          "type": "string"
+        },
+        "ok": {
+          "type": "boolean"
+        },
+        "output": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          ],
+          "description": "The score: one or more labels, or a typed value matching the scorer's lightning type."
+        },
+        "explanation": {
+          "type": "string"
+        },
+        "raw": {
+          "type": "string",
+          "description": "Raw engine output (for debugging / loose formats)."
+        },
+        "error": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "ok",
+        "scorerApiName"
+      ]
+    }
+  }
+}