From 978df218b05465dcdd6075b85f7d077cbcc99a1b Mon Sep 17 00:00:00 2001 From: Ilya Yakelzon Date: Tue, 18 Aug 2026 17:24:03 +0200 Subject: [PATCH 1/2] metrics: export histograms with delta temporality proxy.session.goodput.bucket had grown to ~55M unique series and was timing out every SigNoz query past a ~1w window. The cause is per-host identity on the histogram: host.name alone has 10,854 distinct values, multiplied by 15 bucket boundaries, track, and ~200 client countries. Nothing reads those labels. The experiment evaluator and lantern-dashboard both slice goodput by track and geo.country.iso_code only, and the metric backs 0 dashboards and 0 alerts. They are there because the VPS tags every metric with OTEL_RESOURCE_ATTRIBUTES and SigNoz promotes resource attrs to queryable labels, not because anyone chose per-host goodput. The ops collector can aggregate them away, but only safely on a delta stream: stripping a label from a cumulative stream merges independent monotonic series whose resets interleave, which corrupts rate() silently rather than failing. So switch the exporter to delta for all instrument kinds. Temporality is selected per instrument KIND, so this covers every histogram this binary emits. The only other one with readers is proxy_http_ping_request_duration_seconds, read as a p90 over .bucket by the host_metrics and track_performance_by_volume dashboards; a bucket quantile is computed over per-bucket rates and reads correctly on either temporality. Deploy ordering matters: lantern-cloud reads the paired .count stream with timeAggregation=sum once this ships (delta), where it used increase (cumulative). Mixing them is a ~500x count error, so land the collector and reader changes together and hold experiment_evaluator_autoact_enabled off across the roll. Refs getlantern/engineering#3831 --- otel/otel.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/otel/otel.go b/otel/otel.go index 76a9de5b..499c56ab 100644 --- a/otel/otel.go +++ b/otel/otel.go @@ -171,18 +171,7 @@ func BuildTracerProvider(opts *Opts) (*sdktrace.TracerProvider, func()) { func InitGlobalMeterProvider(opts *Opts) (func(), error) { logExporterEndpoint("metrics") exp, err := otlpmetrichttp.New(context.Background(), - otlpmetrichttp.WithTemporalitySelector(func(kind sdkmetric.InstrumentKind) metricdata.Temporality { - switch kind { - case - sdkmetric.InstrumentKindCounter, - sdkmetric.InstrumentKindUpDownCounter, - sdkmetric.InstrumentKindObservableCounter, - sdkmetric.InstrumentKindObservableUpDownCounter: - return metricdata.DeltaTemporality - default: - return metricdata.CumulativeTemporality - } - }), + otlpmetrichttp.WithTemporalitySelector(deltaTemporality), ) if err != nil { return nil, err @@ -202,3 +191,24 @@ func InitGlobalMeterProvider(opts *Opts) (func(), error) { } }, nil } + +// deltaTemporality exports every instrument kind with delta temporality, +// including histograms. +// +// Delta is what lets the ops collector aggregate an attribute away. Stripping +// a label from a cumulative stream merges independent monotonic series whose +// resets are interleaved, which corrupts rate() silently rather than failing; +// delta datapoints just sum. The ops collector relies on this to drop +// route.id/instance.id/host.name from proxy.session.goodput, whose ~10.8k +// distinct host.name values were driving the histogram to ~55M series (see +// getlantern/engineering#3831). +// +// Temporality is chosen per instrument KIND at the exporter, so this covers +// every histogram this binary emits, not just goodput. The only other one with +// readers is proxy_http_ping_request_duration_seconds, read as a p90 over +// .bucket by the host_metrics and track_performance_by_volume dashboards; a +// bucket quantile is computed over per-bucket rates and reads correctly on +// either temporality. +func deltaTemporality(sdkmetric.InstrumentKind) metricdata.Temporality { + return metricdata.DeltaTemporality +} From 813c88c8c79adad0ee05d7913ddb0d545e6895d9 Mon Sep 17 00:00:00 2001 From: Ilya Yakelzon Date: Tue, 18 Aug 2026 17:44:14 +0200 Subject: [PATCH 2/2] otel: assert histograms export as delta The temporality switch had no test. Mirrors the coverage added to lantern-box after review feedback there: the selector is what makes the ops collector safe to strip route.id/instance.id/host.name from proxy.session.goodput, and it is also what makes lantern-cloud reading the paired .count stream with timeAggregation=sum correct, so a silent revert to cumulative is expensive in both directions. This package had no test file at all, so the assertion is a direct unit test over every instrument kind rather than an export-path integration test. Mutation-checked against a counter-only selector: fails on Histogram/Gauge/UpDownCounter. Refs getlantern/engineering#3831 --- otel/otel_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 otel/otel_test.go diff --git a/otel/otel_test.go b/otel/otel_test.go new file mode 100644 index 00000000..336cc66c --- /dev/null +++ b/otel/otel_test.go @@ -0,0 +1,36 @@ +package otel + +import ( + "testing" + + "github.com/stretchr/testify/assert" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +// TestDeltaTemporalityCoversHistograms guards the exporter's temporality +// selection. +// +// InstrumentKindHistogram is the one that matters: it was cumulative while +// counters were already delta, and it is the kind proxy.session.goodput uses. +// Delta is required for the ops collector to strip +// route.id/instance.id/host.name from that metric — aggregating an identifier +// away merges the matching series, and merging cumulative streams interleaves +// their resets, corrupting rate() with no error. lantern-cloud also reads the +// paired .count stream with timeAggregation=sum, which is only correct for +// delta (increase vs sum on the wrong temporality is a ~500x error). +func TestDeltaTemporalityCoversHistograms(t *testing.T) { + kinds := []sdkmetric.InstrumentKind{ + sdkmetric.InstrumentKindCounter, + sdkmetric.InstrumentKindUpDownCounter, + sdkmetric.InstrumentKindHistogram, + sdkmetric.InstrumentKindGauge, + sdkmetric.InstrumentKindObservableCounter, + sdkmetric.InstrumentKindObservableUpDownCounter, + sdkmetric.InstrumentKindObservableGauge, + } + for _, k := range kinds { + assert.Equal(t, metricdata.DeltaTemporality, deltaTemporality(k), + "instrument kind %v must export delta", k) + } +}