Track dropped extension telemetry reports

cli/azd/.vscode/cspell.yaml

@@ -501,6 +501,10 @@ overrides:
   - filename: internal/cmd/up_graph.go
     words:
       - finalizer
+  - filename: docs/extensions/extension-telemetry.md
+    words:
+      - dcount
+      - isnotempty
 ignorePaths:
   - "**/*_test.go"
   - "**/mock*.go"

cli/azd/cmd/telemetry_test.go

@@ -238,6 +238,14 @@ func TestTelemetryFieldConstants(t *testing.T) {
 		require.Equal(t, "extension.event", string(kv.Key))
 		require.Equal(t, "deploy.completed", kv.Value.AsString())
 
+		dropped := fields.ExtensionUsageDropped.StringSlice(
+			[]string{"contoso.tools@attribute_value_too_long"},
+		)
+		require.Equal(t, "extension.usage.dropped", string(dropped.Key))
+
+		droppedCount := fields.ExtensionUsageDroppedCount.Int64(2)
+		require.Equal(t, "extension.usage.dropped.count", string(droppedCount.Key))
+
 		category := fields.ExtensionSourceCategory.String("local")
 		require.Equal(t, "extension.source.category", string(category.Key))
 

cli/azd/docs/extensions/extension-telemetry.md

@@ -134,7 +134,9 @@ extension author:
 
 Error messages do not echo attribute values. When a valid key has an oversized
 value, the key is included so you can identify the failing field; invalid keys
-are not echoed. Use the status code plus your own call site to diagnose.
+are not echoed. Use the status code plus your own call site to diagnose. Hosts
+that support dropped-report telemetry also record the bounded reason described
+in [Diagnosing dropped reports](#diagnosing-dropped-reports).
 
 ## When your event is not recorded
 
@@ -146,14 +148,75 @@ Two outcomes are deliberately **not** errors. The call succeeds and
 | Your configured source does not match the verified official `azd` registry | Attribute values are never reviewed at runtime, so registry admission is what keeps unchecked content out of `azd`'s pipeline |
 | The per-invocation event budget is spent | `ReportUsage` can be called in a loop, and the per-event bounds do not limit how many events arrive |
 
-Run `azd` with `--debug` to see which one applied.
+Run `azd` with `--debug` to see which one applied. Hosts that support
+dropped-report telemetry also record the bounded reason described in
+[Diagnosing dropped reports](#diagnosing-dropped-reports).
 
 This means **your events are not recorded while you develop locally**, because
 an extension installed with `--source dev` or from a file path does not pass the
 gate. You can still verify your integration end to end: the call succeeds and
 `Accepted` comes back `false`. Because it is not an error, your code runs the
 same path in development as in production — do not branch on `Accepted`.
 
+## Diagnosing dropped reports
+
+`azd` aggregates rejected and dropped calls on the command span that hosted the
+extension. It records two fields:
+
+| Attribute | Meaning |
+|---|---|
+| `extension.usage.dropped` | Unique `<extension-id-or-unattributed>@<reason>` values for the invocation |
+| `extension.usage.dropped.count` | Total reports dropped during the invocation |
+
+The reason is always one of these host-defined values:
+
+| Reason | Cause |
+|---|---|
+| `event_name_invalid` | The event name is missing or exceeds 128 UTF-8 bytes |
+| `attribute_count_exceeded` | The report contains more than 32 attributes |
+| `attribute_key_invalid` | An attribute key is empty or exceeds 128 UTF-8 bytes |
+| `attribute_value_too_long` | An attribute value exceeds 512 UTF-8 bytes |
+| `not_installed` | The calling extension is not installed |
+| `lookup_failed` | `azd` could not read the installed extension record |
+| `source_check_failed` | `azd` could not verify the configured registry source |
+| `source_ineligible` | The configured source is not the verified official registry |
+| `budget_exhausted` | The invocation has already recorded 100 extension usage events |
+| `unauthenticated` | The request did not contain validated extension claims |
+
+No event name, key, value, source, or other caller-controlled content is copied
+into the signal. Repeated failures add to the count but do not add duplicate
+values to the list, so a reporting loop cannot create an unbounded property.
+The extension ID appears only after `azd` verifies the installed record against
+the official registry source. Earlier failures use the fixed value
+`unattributed`.
+
+Use this query to find how many invocations were affected by each extension and
+reason:
+
+```kusto
+requests
+| where isnotempty(tostring(customDimensions["extension.usage.dropped"]))
+| extend dropped = parse_json(tostring(customDimensions["extension.usage.dropped"]))
+| mv-expand dropped
+| extend parts = split(tostring(dropped), "@")
+| summarize affected_invocations=dcount(operation_Id)
+    by extension_id=tostring(parts[0]), reason=tostring(parts[1])
+```
+
+To inspect total volume separately, sum
+`customMeasurements["extension.usage.dropped.count"]`. The fields appear only
+on the command span that hosted the extension; synthetic phase spans created by
+`azd up` do not copy them. The count covers all reasons in an invocation, so do
+not assign it to one reason when the list contains several values.
+
+Accepted `ext.usage` spans and the command span share `operation_Id`. Use that
+field with `extension.id` to compare affected invocations with invocations that
+successfully recorded at least one report.
+
+For a long-running server command, such as `azd vs-server`, the invocation ends
+when the `azd` process exits. Its aggregate therefore covers the process
+lifetime rather than one RPC.
+
 ## Where the data lands
 
 Each accepted event becomes an `ext.usage` span carrying `extension.id`,

cli/azd/internal/cmd/up_graph.go

@@ -192,7 +192,7 @@ func (u *UpGraphAction) Run(
 		// Apply usage attributes (e.g. EnvNameKey) at end so they include
 		// any values set during Run. Globals (e.g. SubscriptionIdKey) are
 		// applied automatically by wrapperSpan.End().
-		usageAttrs := tracing.GetUsageAttributes()
+		usageAttrs := syntheticUpUsageAttributes()
 
 		// Reflect the real package-phase outcome. Before this, the synthetic
 		// span always closed with an Unset status (=> Success in the AppInsights
@@ -800,6 +800,16 @@ func changedFlagNames(fs *pflag.FlagSet) []string {
 	return names
 }
 
+// syntheticUpUsageAttributes returns usage attributes for the synthetic
+// cmd.package, cmd.provision, and cmd.deploy spans. Extension drop attributes
+// stay on the hosting cmd.up span so invocation totals are not duplicated.
+func syntheticUpUsageAttributes() []attribute.KeyValue {
+	return slices.DeleteFunc(tracing.GetUsageAttributes(), func(attr attribute.KeyValue) bool {
+		return attr.Key == fields.ExtensionUsageDropped.Key ||
+			attr.Key == fields.ExtensionUsageDroppedCount.Key
+	})
+}
+
 // initializeServices enumerates services, initializes the project, and ensures
 // that required service target tools are available.
 func (u *UpGraphAction) initializeServices(ctx context.Context) ([]*project.ServiceConfig, error) {

cli/azd/internal/cmd/up_graph_telemetry_test.go

@@ -10,10 +10,13 @@ import (
 	"testing"
 	"time"
 
+	"github.com/azure/azure-dev/cli/azd/internal/tracing"
+	"github.com/azure/azure-dev/cli/azd/internal/tracing/fields"
 	"github.com/azure/azure-dev/cli/azd/pkg/exegraph"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
 	"go.opentelemetry.io/otel/sdk/trace/tracetest"
@@ -638,6 +641,28 @@ func TestEmitDeploySpan_ErrorPath(t *testing.T) {
 	assert.True(t, end.Equal(span.EndTime()), "end = %s, want %s", span.EndTime(), end)
 }
 
+func TestSyntheticUpUsageAttributesExcludeExtensionDrops(t *testing.T) {
+	tracing.ResetUsageAttributesForTest()
+	t.Cleanup(tracing.ResetUsageAttributesForTest)
+
+	const retainedKey = attribute.Key("test.retained")
+	tracing.SetUsageAttributes(
+		retainedKey.String("value"),
+		fields.ExtensionUsageDropped.StringSlice([]string{"publisher.extension@budget_exhausted"}),
+		fields.ExtensionUsageDroppedCount.Int64(1),
+	)
+
+	attrs := syntheticUpUsageAttributes()
+	indexed := map[attribute.Key]attribute.Value{}
+	for _, attr := range attrs {
+		indexed[attr.Key] = attr.Value
+	}
+
+	require.Contains(t, indexed, retainedKey)
+	assert.NotContains(t, indexed, fields.ExtensionUsageDropped.Key)
+	assert.NotContains(t, indexed, fields.ExtensionUsageDroppedCount.Key)
+}
+
 // TestEmitDeploySpan_OmittedWhenDeployDidNotRun verifies the other half of the
 // contract: when provisioning fails first and the deploy phase is skipped before
 // it starts, no cmd.deploy span is emitted — matching legacy `azd up`, where the

cli/azd/internal/grpcserver/server.go

@@ -235,6 +235,9 @@ func (s *Server) tokenAuthInterceptor(serverInfo *ServerInfo) grpc.UnaryServerIn
 	) (any, error) {
 		ctx, err := s.validateAuthToken(ctx, serverInfo)
 		if err != nil {
+			if info.FullMethod == azdext.TelemetryService_ReportUsage_FullMethodName {
+				recordExtensionUsageDrop(unattributedExtensionId, extensionUsageDropReasonUnauthenticated)
+			}
 			return nil, err
 		}
 

cli/azd/internal/grpcserver/server_test.go

@@ -26,6 +26,7 @@ import (
 	"google.golang.org/protobuf/proto"
 
 	"github.com/azure/azure-dev/cli/azd/internal"
+	"github.com/azure/azure-dev/cli/azd/internal/tracing"
 	"github.com/azure/azure-dev/cli/azd/pkg/account"
 	"github.com/azure/azure-dev/cli/azd/pkg/auth"
 	"github.com/azure/azure-dev/cli/azd/pkg/azapi"
@@ -187,6 +188,9 @@ func Test_Server_Start(t *testing.T) {
 	})
 
 	t.Run("TelemetryMissingToken", func(t *testing.T) {
+		tracing.ResetUsageAttributesForTest()
+		t.Cleanup(tracing.ResetUsageAttributesForTest)
+
 		client, err := azdext.NewAzdClient(azdext.WithAddress(serverInfo.Address))
 		require.NoError(t, err)
 
@@ -197,6 +201,8 @@ func Test_Server_Start(t *testing.T) {
 		st, ok := status.FromError(err)
 		require.True(t, ok)
 		require.Equal(t, codes.Unauthenticated, st.Code())
+		requireUsageDrop(t, unattributedExtensionId,
+			[]extensionUsageDropReason{extensionUsageDropReasonUnauthenticated}, 1)
 	})
 }
 

cli/azd/internal/grpcserver/telemetry_service.go

@@ -43,6 +43,23 @@ const (
 	maxUsageEventsPerInvocation = 100
 )
 
+type extensionUsageDropReason string
+
+const (
+	extensionUsageDropReasonEventNameInvalid       extensionUsageDropReason = "event_name_invalid"
+	extensionUsageDropReasonAttributeCountExceeded extensionUsageDropReason = "attribute_count_exceeded"
+	extensionUsageDropReasonAttributeKeyInvalid    extensionUsageDropReason = "attribute_key_invalid"
+	extensionUsageDropReasonAttributeValueTooLong  extensionUsageDropReason = "attribute_value_too_long"
+	extensionUsageDropReasonNotInstalled           extensionUsageDropReason = "not_installed"
+	extensionUsageDropReasonLookupFailed           extensionUsageDropReason = "lookup_failed"
+	extensionUsageDropReasonSourceCheckFailed      extensionUsageDropReason = "source_check_failed"
+	extensionUsageDropReasonSourceIneligible       extensionUsageDropReason = "source_ineligible"
+	extensionUsageDropReasonBudgetExhausted        extensionUsageDropReason = "budget_exhausted"
+	extensionUsageDropReasonUnauthenticated        extensionUsageDropReason = "unauthenticated"
+
+	unattributedExtensionId = "unattributed"
+)
+
 // installedExtensionLookup resolves the installed extension record for a
 // signed extension id. *extensions.Manager satisfies it.
 type installedExtensionLookup interface {
@@ -92,38 +109,44 @@ func (s *telemetryService) ReportUsage(
 ) (*azdext.ReportUsageResponse, error) {
 	claims, err := extensions.GetClaimsFromContext(ctx)
 	if err != nil {
+		recordExtensionUsageDrop(unattributedExtensionId, extensionUsageDropReasonUnauthenticated)
 		return nil, status.Error(codes.Unauthenticated, "validated extension claims are required")
 	}
 
-	if err := validateUsageRequest(req); err != nil {
-		return nil, err
-	}
-
 	extension, err := s.extensions.GetInstalled(extensions.FilterOptions{Id: claims.Subject})
 	if err != nil {
 		if errors.Is(err, extensions.ErrInstalledExtensionNotFound) {
+			recordExtensionUsageDrop(unattributedExtensionId, extensionUsageDropReasonNotInstalled)
 			return nil, status.Error(codes.PermissionDenied, "extension is not installed")
 		}
 
+		recordExtensionUsageDrop(unattributedExtensionId, extensionUsageDropReasonLookupFailed)
 		return nil, status.Error(codes.Internal, "failed to verify installed extension")
 	}
 
 	official, err := s.extensions.IsOfficialRegistrySource(ctx, extension.Source)
 	if err != nil {
+		recordExtensionUsageDrop(unattributedExtensionId, extensionUsageDropReasonSourceCheckFailed)
 		log.Printf(
 			"telemetry: failed to verify source %q for %s: %v",
 			extension.Source, extension.Id, err)
 		return &azdext.ReportUsageResponse{Accepted: false}, nil
 	}
 
 	if !official {
+		recordExtensionUsageDrop(unattributedExtensionId, extensionUsageDropReasonSourceIneligible)
 		log.Printf(
 			"telemetry: dropping usage event from %s installed from source %q",
 			extension.Id, extension.Source)
 
 		return &azdext.ReportUsageResponse{Accepted: false}, nil
 	}
 
+	if reason, err := validateUsageRequest(req); err != nil {
+		recordExtensionUsageDrop(extension.Id, reason)
+		return nil, err
+	}
+
 	attributes := []attribute.KeyValue{
 		fields.ExtensionId.String(extension.Id),
 		fields.ExtensionVersion.String(extension.Version),
@@ -141,6 +164,7 @@ func (s *telemetryService) ReportUsage(
 	// a container singleton, which makes it a per-invocation budget
 	// even when a composite command such as up runs several actions.
 	if s.recorded.Add(1) > maxUsageEventsPerInvocation {
+		recordExtensionUsageDrop(extension.Id, extensionUsageDropReasonBudgetExhausted)
 		log.Printf(
 			"telemetry: dropping usage event %q from %s, limit of %d reached",
 			req.EventName, extension.Id, maxUsageEventsPerInvocation)
@@ -158,31 +182,40 @@ func (s *telemetryService) ReportUsage(
 	return &azdext.ReportUsageResponse{Accepted: true}, nil
 }
 
-func validateUsageRequest(req *azdext.ReportUsageRequest) error {
+func recordExtensionUsageDrop(extensionId string, reason extensionUsageDropReason) {
+	tracing.AppendUsageAttributeUnique(
+		fields.ExtensionUsageDropped.String(extensionId + "@" + string(reason)),
+	)
+	tracing.IncrementUsageAttribute(fields.ExtensionUsageDroppedCount.Int64(1))
+}
+
+func validateUsageRequest(req *azdext.ReportUsageRequest) (extensionUsageDropReason, error) {
 	if req == nil || req.EventName == "" || len(req.EventName) > maxUsageEventNameBytes {
-		return status.Errorf(codes.InvalidArgument,
+		return extensionUsageDropReasonEventNameInvalid, status.Errorf(codes.InvalidArgument,
 			"event name is required and must be at most %d UTF-8 bytes",
 			maxUsageEventNameBytes)
 	}
 
 	if len(req.Attributes) > maxUsageAttributes {
-		return status.Errorf(codes.InvalidArgument,
+		return extensionUsageDropReasonAttributeCountExceeded, status.Errorf(codes.InvalidArgument,
 			"event declares more than %d attributes", maxUsageAttributes)
 	}
 
-	for key, value := range req.Attributes {
+	for key := range req.Attributes {
 		if key == "" || len(key) > maxUsageKeyBytes {
-			return status.Errorf(codes.InvalidArgument,
+			return extensionUsageDropReasonAttributeKeyInvalid, status.Errorf(codes.InvalidArgument,
 				"attribute keys are required and must be at most %d UTF-8 bytes",
 				maxUsageKeyBytes)
 		}
+	}
 
+	for key, value := range req.Attributes {
 		if len(value) > maxUsageValueBytes {
-			return status.Errorf(codes.InvalidArgument,
+			return extensionUsageDropReasonAttributeValueTooLong, status.Errorf(codes.InvalidArgument,
 				"attribute value for key %q must be at most %d UTF-8 bytes",
 				key, maxUsageValueBytes)
 		}
 	}
 
-	return nil
+	return "", nil
 }