From d4a2e51cfa2cf9953954bf59f77c178b8fbde2c0 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sun, 30 Aug 2026 02:15:06 +0200 Subject: [PATCH] fix(integrations): make a fresh Cloudflare connection explain itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connecting Cloudflare did no work: the callback stored the grant and returned, leaving discovery and the first poll to the alerting worker's */5 cron. Between that cadence and the 10-minute safety lag on incomplete analytics buckets, a newly connected org saw an account name and nothing else — no zones, no Workers, no data — for 10-20 minutes, on every Cloudflare surface, with no copy anywhere saying that was expected or when it would end. Prime the org inline instead: the callback now runs pollOrg after resetOrgState (which has already cleared discoveredAt), bounded by a 12s timeout and best-effort, so discovery commits before the popup reports success and whatever polling the timeout cuts short resumes on the next tick. Then say what is happening while the rest arrives. Status gains connectedAt (from the grant row's createdAt, which a token refresh never touches) and a per-zone backfillAt, which was already in Postgres; cloudflareIngestPhase() turns those plus warehouse usage into one of discovering / collecting / partial / backfilling / live / stalled, and every Cloudflare surface — the infra index, the zone drill-in, the integrations card — renders its banner and empty state from that single derivation, so they cannot disagree about whether an empty page is normal. Status and usage poll every 30s until the phase is live, so the page fills in without a reload. Backfill progress reads the zone with the HIGHEST frontier: the frontier walks down, so that is the one furthest from done, and a day-wide chart is only as complete as it. Reading the lowest would let one finished zone claim the whole account was caught up. Silence past 30 minutes stops being described as a cadence and starts pointing at traffic and permissions instead. --- apps/api/src/routes/v1/integrations.http.ts | 28 +++ .../services/auth/CloudflareOAuthService.ts | 16 +- .../CloudflareAnalyticsService.test.ts | 1 + .../CloudflareAnalyticsService.ts | 6 + .../cloudflare/cloudflare-ingest-status.tsx | 84 ++++++++ .../infra/cloudflare/ingest-phase.test.ts | 200 ++++++++++++++++++ .../infra/cloudflare/ingest-phase.ts | 157 ++++++++++++++ .../cloudflare/use-cloudflare-ingest-phase.ts | 53 +++++ .../integrations/cloudflare-account-card.tsx | 31 ++- .../src/routes/infra/cloudflare/$zoneName.tsx | 15 ++ .../web/src/routes/infra/cloudflare/index.tsx | 52 +++-- packages/domain/src/http/integrations.ts | 14 ++ 12 files changed, 626 insertions(+), 31 deletions(-) create mode 100644 apps/web/src/components/infra/cloudflare/cloudflare-ingest-status.tsx create mode 100644 apps/web/src/components/infra/cloudflare/ingest-phase.test.ts create mode 100644 apps/web/src/components/infra/cloudflare/ingest-phase.ts create mode 100644 apps/web/src/components/infra/cloudflare/use-cloudflare-ingest-phase.ts diff --git a/apps/api/src/routes/v1/integrations.http.ts b/apps/api/src/routes/v1/integrations.http.ts index e77a4d9fd..e6514013c 100644 --- a/apps/api/src/routes/v1/integrations.http.ts +++ b/apps/api/src/routes/v1/integrations.http.ts @@ -79,6 +79,13 @@ const GITHUB_MESSAGE_TYPE = "maple:integration:github" const CLOUDFLARE_MESSAGE_TYPE = "maple:integration:cloudflare" const PLANETSCALE_MESSAGE_TYPE = "maple:integration:planetscale" +/** + * How long the Cloudflare callback waits on the post-connect prime poll. Long enough for zone + * discovery plus a first window on an ordinary account, short enough that a slow or many-zoned + * one still gets its success page promptly — the cron finishes whatever is left. + */ +const CLOUDFLARE_PRIME_TIMEOUT = "12 seconds" + const resolveRequestOrigin = (req: HttpServerRequest.HttpServerRequest): string => { const headers = req.headers as Record const forwardedHost = headers["x-forwarded-host"] @@ -1091,6 +1098,27 @@ export const IntegrationsCallbackRouter = HttpRouter.use((router) => ), ), ), + // Prime the org before the popup reports success. Without this the integration is + // entirely blank — no zones, no Workers, no data — until the alerting cron's next + // */5 tick discovers them, which reads as "connecting did nothing". `resetOrgState` + // above cleared `discoveredAt`, so this poll rediscovers and then spends what call + // budget it has on the newest window. + // + // Bounded and best-effort: discovery (the part that makes the UI stop looking + // empty) commits in the first seconds, and whatever polling the timeout cuts short + // resumes on the next tick — a lease released by interruption or expiry, never a + // failed callback page. + Effect.tap((result) => + cloudflareAnalytics.pollOrg(result.orgId).pipe( + Effect.timeout(CLOUDFLARE_PRIME_TIMEOUT), + Effect.catchCause((cause) => + Effect.logWarning("cloudflare post-connect prime poll incomplete", { + orgId: result.orgId, + error: summarizeCause(cause), + }), + ), + ), + ), Effect.map((result) => htmlResponse( cloudflareCallbackPage({ diff --git a/apps/api/src/services/auth/CloudflareOAuthService.ts b/apps/api/src/services/auth/CloudflareOAuthService.ts index 60326fcd2..33e816d02 100644 --- a/apps/api/src/services/auth/CloudflareOAuthService.ts +++ b/apps/api/src/services/auth/CloudflareOAuthService.ts @@ -14,7 +14,7 @@ import { FetchHttpClient } from "effect/unstable/http" import { listAccounts } from "@/services/integrations/CloudflareApi" import { Database } from "@/platform/DatabaseLive" import { Env, type EnvConfig } from "@/platform/Env" -import { msToDate } from "@/platform/time" +import { dateToMs, msToDate } from "@/platform/time" import { makeOAuthConnectionHelpers, OAUTH_STATE_TTL_MS } from "./oauth/connection-helpers" const CLOUDFLARE_PROVIDER = "cloudflare" @@ -126,9 +126,14 @@ export interface CloudflareConnectedAccount { * own principal, first in grant order) without asserting an index is there. */ export type CloudflareConnectionStatus = - | { readonly connected: false; readonly accounts: readonly [] } + | { readonly connected: false; readonly accounts: readonly []; readonly connectedAt: null } | { readonly connected: true + /** + * When the grant row was created (epoch ms). A reconnect over a live grant keeps it, + * and a token refresh never touches it — so it dates the connection, not the token. + */ + readonly connectedAt: number readonly accounts: Arr.NonEmptyReadonlyArray } @@ -364,10 +369,15 @@ export class CloudflareOAuthService extends Context.Service< const getStatus = Effect.fn("CloudflareOAuthService.getStatus")(function* (orgId: OrgId) { const row = yield* oauth.loadConnection(orgId) if (!row) { - return { connected: false, accounts: [] } satisfies CloudflareConnectionStatus + return { + connected: false, + accounts: [], + connectedAt: null, + } satisfies CloudflareConnectionStatus } return { connected: true, + connectedAt: dateToMs(row.createdAt), // `Arr.map` carries the non-emptiness through, so the connected branch keeps its // at-least-one-account guarantee. accounts: Arr.map( diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts index ec53531c1..f79b089dd 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.test.ts @@ -1536,6 +1536,7 @@ describe("CloudflareAnalyticsService", () => { lastSyncedAt: T0 - 5 * MIN, lastError: null, watermarkAt: T0 - 15 * MIN, + backfillAt: null, }) assert.strictEqual(status.workers?.lastError, "boom") assert.strictEqual(status.workers?.watermarkAt, null) diff --git a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts index 109d6f1b7..cf531738e 100644 --- a/apps/api/src/services/integrations/CloudflareAnalyticsService.ts +++ b/apps/api/src/services/integrations/CloudflareAnalyticsService.ts @@ -1011,6 +1011,7 @@ interface CloudflareAnalyticsZoneStatusFields { readonly lastSyncedAt: number | null readonly lastError: string | null readonly watermarkAt: number | null + readonly backfillAt: number | null } interface CloudflareAnalyticsWorkersStatusFields { @@ -1018,6 +1019,7 @@ interface CloudflareAnalyticsWorkersStatusFields { readonly lastSyncedAt: number | null readonly lastError: string | null readonly watermarkAt: number | null + readonly backfillAt: number | null } /** One connected account's collection state. */ @@ -2412,12 +2414,14 @@ export class CloudflareAnalyticsService extends Context.Service< lastSyncedAt: row.lastSuccessAt == null ? null : dateToMs(row.lastSuccessAt), lastError: row.lastError, watermarkAt: row.watermarkAt == null ? null : dateToMs(row.watermarkAt), + backfillAt: row.backfillAt == null ? null : dateToMs(row.backfillAt), }) const toWorkers = (row: CloudflareAnalyticsStateRow): CloudflareAnalyticsWorkersStatusFields => ({ enabled: row.enabled, lastSyncedAt: row.lastSuccessAt == null ? null : dateToMs(row.lastSuccessAt), lastError: row.lastError, watermarkAt: row.watermarkAt == null ? null : dateToMs(row.watermarkAt), + backfillAt: row.backfillAt == null ? null : dateToMs(row.backfillAt), }) const accountIds = [...new Set(rows.map((row) => row.accountId))].sort() const accounts = accountIds.map((accountId): CloudflareAnalyticsAccountStatus => { @@ -2611,6 +2615,7 @@ export class CloudflareAnalyticsService extends Context.Service< connectedByUserId: null, scope: null, analyticsCapable: false, + connectedAt: null, accounts: [], zones: [], workers: null, @@ -2652,6 +2657,7 @@ export class CloudflareAnalyticsService extends Context.Service< connectedByUserId: decodeUserIdSync(primary.connectedByUserId), scope: primary.scope, analyticsCapable: accounts.some((account) => account.analyticsCapable && !account.revoked), + connectedAt: connection.connectedAt, accounts, zones: mergedZones, workers: mergedWorkers, diff --git a/apps/web/src/components/infra/cloudflare/cloudflare-ingest-status.tsx b/apps/web/src/components/infra/cloudflare/cloudflare-ingest-status.tsx new file mode 100644 index 000000000..d2022d8db --- /dev/null +++ b/apps/web/src/components/infra/cloudflare/cloudflare-ingest-status.tsx @@ -0,0 +1,84 @@ +import { Link } from "@tanstack/react-router" + +import { Alert, AlertDescription, AlertTitle } from "@maple/ui/components/ui/alert" +import { Button } from "@maple/ui/components/ui/button" +import { + Empty, + EmptyContent, + EmptyDescription, + EmptyHeader, + EmptyMedia, + EmptyTitle, +} from "@maple/ui/components/ui/empty" + +import { CircleInfoIcon, CircleWarningIcon, CloudflareIcon, LoaderIcon } from "@/components/icons" +import { describeCloudflareIngestPhase, type CloudflareIngestPhase } from "./ingest-phase" + +/** + * The in-progress phases of a healthy new connection. They share one presentation — a spinner and + * an explanation of when data lands — because to the reader they are one thing: "it's coming". + */ +const isWorking = (phase: CloudflareIngestPhase): boolean => + phase.kind === "discovering" || phase.kind === "collecting" || phase.kind === "backfilling" + +/** + * Explains an empty or half-empty Cloudflare page, above whatever data already exists. Renders + * nothing once everything is live — a banner that says "working" forever is just chrome. + */ +export function CloudflareIngestBanner({ phase }: { phase: CloudflareIngestPhase }) { + if (phase.kind === "live") return null + const { title, description, tone } = describeCloudflareIngestPhase(phase) + return ( + + {isWorking(phase) ? ( + + ) : tone === "warning" ? ( + + ) : ( + + )} + {title} + {description} + + ) +} + +/** + * Full-page stand-in when a Cloudflare surface has nothing to draw yet. Same copy as the banner, + * so a page that starts empty and later fills in never contradicts itself. + */ +export function CloudflareIngestEmpty({ + phase, + children, +}: { + phase: CloudflareIngestPhase + /** Optional action row — e.g. a link back to the integration when something needs fixing. */ + children?: React.ReactNode +}) { + const { title, description } = describeCloudflareIngestPhase(phase) + return ( + + + + {isWorking(phase) ? ( + + ) : ( + + )} + + {title} + {description} + + {children ? {children} : null} + + ) +} + +/** The one action a stalled connection has: re-grant access from the integrations page. */ +export function CloudflareStalledAction() { + return ( + + ) +} diff --git a/apps/web/src/components/infra/cloudflare/ingest-phase.test.ts b/apps/web/src/components/infra/cloudflare/ingest-phase.test.ts new file mode 100644 index 000000000..56980e21c --- /dev/null +++ b/apps/web/src/components/infra/cloudflare/ingest-phase.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from "vitest" + +import { + CloudflareAnalyticsWorkersStatus, + CloudflareAnalyticsZoneStatus, + CloudflareIntegrationStatus, + CloudflareServiceUsage, + CloudflareUsageResponse, +} from "@maple/domain/http" + +import { cloudflareIngestPhase } from "./ingest-phase" + +const NOW = 1_800_000_000_000 +const MINUTE = 60_000 +const HOUR = 60 * MINUTE + +const zone = ( + name: string, + overrides: Partial<{ watermarkAt: number | null; backfillAt: number | null; enabled: boolean }> = {}, +) => + new CloudflareAnalyticsZoneStatus({ + id: `zone-${name}`, + name, + enabled: overrides.enabled ?? true, + lastSyncedAt: null, + lastError: null, + watermarkAt: overrides.watermarkAt ?? null, + backfillAt: overrides.backfillAt ?? null, + }) + +const status = (overrides: { + zones?: ReadonlyArray + workers?: CloudflareAnalyticsWorkersStatus | null + connectedAt?: number | null +}) => + new CloudflareIntegrationStatus({ + connected: true, + accountId: "acct", + accountName: "Acme", + connectedByUserId: null, + scope: "analytics.read", + analyticsCapable: true, + connectedAt: overrides.connectedAt ?? NOW - 2 * MINUTE, + accounts: [], + zones: overrides.zones ?? [], + workers: overrides.workers ?? null, + }) + +const usage = (services: ReadonlyArray<{ name: string; kind: "zone" | "worker"; requests: number }>) => + new CloudflareUsageResponse({ + windowStart: NOW - 24 * HOUR, + windowEnd: NOW, + bucketSeconds: 3600, + totalRequests: services.reduce((sum, service) => sum + service.requests, 0), + services: services.map( + (service) => + new CloudflareServiceUsage({ + serviceName: `cloudflare/${service.name}`, + kind: service.kind, + displayName: service.name, + totalRequests: service.requests, + totalDatapoints: service.requests, + lastDataAt: service.requests > 0 ? NOW - MINUTE : null, + buckets: [], + }), + ), + }) + +describe("cloudflareIngestPhase", () => { + it("reports discovery while the poller has no state rows yet", () => { + expect(cloudflareIngestPhase(status({}), usage([]), NOW)).toEqual({ kind: "discovering" }) + }) + + it("reports collecting once zones are known but nothing is queryable", () => { + const result = cloudflareIngestPhase(status({ zones: [zone("acme.com")] }), usage([]), NOW) + expect(result).toEqual({ kind: "collecting" }) + }) + + // The distinction the whole feature exists for: a young silence is the cadence, an old one + // is a fault, and only the second one should tell someone to go check their connection. + it("calls a long silence stalled rather than collecting", () => { + const result = cloudflareIngestPhase( + status({ zones: [zone("acme.com")], connectedAt: NOW - 45 * MINUTE }), + usage([]), + NOW, + ) + expect(result).toEqual({ kind: "stalled" }) + }) + + it("stalls a connection that never discovered anything", () => { + expect(cloudflareIngestPhase(status({ connectedAt: NOW - 45 * MINUTE }), usage([]), NOW)).toEqual({ + kind: "stalled", + }) + }) + + it("holds the poller's view while usage is unavailable", () => { + const collecting = cloudflareIngestPhase(status({ zones: [zone("acme.com")] }), null, NOW) + expect(collecting).toEqual({ kind: "collecting" }) + + const live = cloudflareIngestPhase( + status({ zones: [zone("acme.com", { watermarkAt: NOW - 11 * MINUTE })] }), + null, + NOW, + ) + expect(live).toEqual({ kind: "live" }) + }) + + it("reports partial coverage while some zones are still catching up", () => { + const result = cloudflareIngestPhase( + status({ zones: [zone("acme.com"), zone("quiet.com")] }), + usage([{ name: "acme.com", kind: "zone", requests: 10 }]), + NOW, + ) + expect(result).toEqual({ kind: "partial", live: 1, total: 2 }) + }) + + // Past the stall window a zone with no traffic is just quiet — nagging about it forever + // would make the banner permanent furniture on any account with an idle domain. + it("stops calling a long-quiet zone partial", () => { + const result = cloudflareIngestPhase( + status({ + zones: [zone("acme.com"), zone("quiet.com")], + connectedAt: NOW - 45 * MINUTE, + }), + usage([{ name: "acme.com", kind: "zone", requests: 10 }]), + NOW, + ) + expect(result).toEqual({ kind: "live" }) + }) + + it("ignores disabled zones when judging coverage", () => { + const result = cloudflareIngestPhase( + status({ zones: [zone("acme.com"), zone("off.com", { enabled: false })] }), + usage([{ name: "acme.com", kind: "zone", requests: 10 }]), + NOW, + ) + expect(result).toEqual({ kind: "live" }) + }) + + it("counts Workers data as live even with no zones reporting", () => { + const workers = new CloudflareAnalyticsWorkersStatus({ + enabled: true, + lastSyncedAt: NOW - MINUTE, + lastError: null, + watermarkAt: NOW - 11 * MINUTE, + backfillAt: null, + }) + const result = cloudflareIngestPhase( + status({ workers }), + usage([{ name: "api", kind: "worker", requests: 5 }]), + NOW, + ) + expect(result).toEqual({ kind: "live" }) + }) + + it("reports backfill progress from the least caught-up zone", () => { + const result = cloudflareIngestPhase( + status({ + zones: [ + zone("acme.com", { backfillAt: NOW - 20 * HOUR }), + zone("other.com", { backfillAt: NOW - 6 * HOUR }), + ], + }), + usage([ + { name: "acme.com", kind: "zone", requests: 10 }, + { name: "other.com", kind: "zone", requests: 10 }, + ]), + NOW, + ) + // other.com has only walked back 6h, so 18 of the 24 hours are still missing from a + // day-wide chart — the caught-up zone next to it must not round that away. + expect(result).toEqual({ kind: "backfilling", progress: 6 / 24 }) + }) + + it("stays in backfill while one zone lags a finished one", () => { + const result = cloudflareIngestPhase( + status({ + zones: [ + zone("done.com", { backfillAt: NOW - 24 * HOUR }), + zone("lagging.com", { backfillAt: NOW - 3 * HOUR }), + ], + }), + usage([ + { name: "done.com", kind: "zone", requests: 10 }, + { name: "lagging.com", kind: "zone", requests: 10 }, + ]), + NOW, + ) + expect(result).toEqual({ kind: "backfilling", progress: 3 / 24 }) + }) + + it("treats a frontier within a poll window of the floor as finished", () => { + const result = cloudflareIngestPhase( + status({ zones: [zone("acme.com", { backfillAt: NOW - 23.5 * HOUR })] }), + usage([{ name: "acme.com", kind: "zone", requests: 10 }]), + NOW, + ) + expect(result).toEqual({ kind: "live" }) + }) +}) diff --git a/apps/web/src/components/infra/cloudflare/ingest-phase.ts b/apps/web/src/components/infra/cloudflare/ingest-phase.ts new file mode 100644 index 000000000..2b78fed0a --- /dev/null +++ b/apps/web/src/components/infra/cloudflare/ingest-phase.ts @@ -0,0 +1,157 @@ +import type { CloudflareIntegrationStatus, CloudflareUsageResponse } from "@maple/domain/http" + +/** + * How far a connected Cloudflare integration is from showing real numbers. Every Cloudflare + * surface derives its banner and empty state from this one function so they can never disagree + * about whether an empty page is normal or broken. + * + * The poller's shape is what makes the wait explainable: the alerting cron ticks every 5 minutes + * and never queries buckets younger than the 10-minute safety lag, so a fresh connection is empty + * for a quarter hour by design — and the 24h history then fills in behind it, bounded by the + * per-tick call budget. Silence past {@link STALL_AFTER_MS} is no longer explainable that way. + */ +export type CloudflareIngestPhase = + /** Connected, but the poller hasn't listed the account's zones and Workers yet. */ + | { readonly kind: "discovering" } + /** Zones known, nothing queryable yet — the expected first ~15 minutes. */ + | { readonly kind: "collecting" } + /** Some zones report data, others don't. */ + | { readonly kind: "partial"; readonly live: number; readonly total: number } + /** Everything live, history still filling. `progress` is the fraction of the 24h window done. */ + | { readonly kind: "backfilling"; readonly progress: number } + | { readonly kind: "live" } + /** Connected long enough that "give it a few minutes" has stopped being the answer. */ + | { readonly kind: "stalled" } + +/** Cloudflare batches analytics in 5-minute buckets; the poller reads them on the same cadence. */ +export const POLL_INTERVAL_MINUTES = 5 +/** Buckets younger than this are incomplete, so the poller never asks for them. */ +export const SAFETY_LAG_MINUTES = 10 +/** What we promise a freshly-connected org: cadence + lag, rounded up to a whole number. */ +export const FIRST_DATA_MINUTES = POLL_INTERVAL_MINUTES + SAFETY_LAG_MINUTES + +/** Past this with nothing ingested, the wait is a problem rather than a cadence. */ +const STALL_AFTER_MS = 30 * 60_000 +/** The history the poller backfills, matching `BACKFILL_MS` in `CloudflareAnalyticsService`. */ +const BACKFILL_WINDOW_MS = 24 * 60 * 60_000 +/** Backfill within one poll window of the floor is finished for display purposes. */ +const BACKFILL_DONE_SLACK_MS = 60 * 60_000 + +const hasData = (usage: CloudflareUsageResponse, displayName: string): boolean => + usage.services.some( + (service) => + service.displayName === displayName && (service.totalRequests > 0 || service.lastDataAt != null), + ) + +/** + * `usage` is the warehouse read — pass `null` while it is loading or failed, which holds the + * phase at the poller's own view rather than reporting an absence the warehouse never confirmed. + */ +export function cloudflareIngestPhase( + status: CloudflareIntegrationStatus, + usage: CloudflareUsageResponse | null, + now: number, +): CloudflareIngestPhase { + const zones = status.zones.filter((zone) => zone.enabled) + const stalled = status.connectedAt != null && now - status.connectedAt > STALL_AFTER_MS + + // No state rows at all: discovery hasn't run (or found nothing). The connect callback primes + // it, so this is normally a few seconds — long enough and it's the same dead end as no data. + if (zones.length === 0 && status.workers == null) { + return stalled ? { kind: "stalled" } : { kind: "discovering" } + } + + // Usage in flight: report the poller's own progress rather than guessing at the warehouse. + if (usage == null) { + const anySynced = + zones.some((zone) => zone.watermarkAt != null) || status.workers?.watermarkAt != null + return anySynced ? { kind: "live" } : { kind: "collecting" } + } + + const live = zones.filter((zone) => hasData(usage, zone.name)).length + const workersLive = + status.workers != null && + usage.services.some( + (service) => + service.kind === "worker" && (service.totalRequests > 0 || service.lastDataAt != null), + ) + + if (live === 0 && !workersLive) { + return stalled ? { kind: "stalled" } : { kind: "collecting" } + } + // A zone with no traffic at all is indistinguishable from one still catching up, so only + // call it partial while the connection is young enough for catching-up to be the likelier + // explanation — past that, a quiet zone is just quiet and shouldn't nag forever. + if (live < zones.length && !stalled) { + return { kind: "partial", live, total: zones.length } + } + + const progress = backfillProgress(status, now) + return progress == null ? { kind: "live" } : { kind: "backfilling", progress } +} + +/** + * Fraction of the 24h history already ingested, or null when it is complete (or hasn't started — + * the frontier is seeded by the first head poll). Zones fill independently and the frontier walks + * DOWN, so the zone with the HIGHEST frontier is the one furthest from done — and a chart reaching + * back a day is only as complete as that zone. Reporting the lowest instead would let one + * finished zone (whose frontier rests at the floor) claim the whole account was caught up. + */ +function backfillProgress(status: CloudflareIntegrationStatus, now: number): number | null { + const frontiers = [ + ...status.zones.filter((zone) => zone.enabled).map((zone) => zone.backfillAt), + status.workers?.backfillAt, + ].filter((value): value is number => value != null) + if (frontiers.length === 0) return null + const leastCaughtUp = Math.max(...frontiers) + const floor = now - BACKFILL_WINDOW_MS + const remaining = leastCaughtUp - floor + if (remaining <= BACKFILL_DONE_SLACK_MS) return null + return Math.min(1, Math.max(0, 1 - remaining / BACKFILL_WINDOW_MS)) +} + +/** Banner/empty copy for a phase. One place, so every surface says the same thing. */ +export function describeCloudflareIngestPhase(phase: CloudflareIngestPhase): { + readonly title: string + readonly description: string + readonly tone: "info" | "warning" +} { + switch (phase.kind) { + case "discovering": + return { + title: "Finding your zones and Workers", + description: + "Maple is listing everything the connected Cloudflare accounts cover. This usually takes a few seconds.", + tone: "info", + } + case "collecting": + return { + title: "Collecting your first Cloudflare data", + description: `Cloudflare publishes analytics in ${POLL_INTERVAL_MINUTES}-minute batches and needs about ${SAFETY_LAG_MINUTES} minutes before a batch is complete, so the first numbers usually land within ${FIRST_DATA_MINUTES} minutes of connecting. This page updates on its own.`, + tone: "info", + } + case "partial": + return { + title: `${phase.live} of ${phase.total} zones reporting`, + description: + "The rest are still catching up, or had no traffic in this window. Nothing to do — they fill in as the poller works through them.", + tone: "info", + } + case "backfilling": + return { + title: `Backfilling history — ${Math.round(phase.progress * 100)}% of the last 24 hours`, + description: + "Live data is already flowing. Older windows arrive a few at a time, so charts reaching further back keep filling in.", + tone: "info", + } + case "stalled": + return { + title: "No Cloudflare data has arrived", + description: + "Collection has been connected for a while with nothing ingested. Check that the zones have traffic, and that the connection still has the analytics permissions — reconnecting re-grants them.", + tone: "warning", + } + case "live": + return { title: "Receiving Cloudflare data", description: "", tone: "info" } + } +} diff --git a/apps/web/src/components/infra/cloudflare/use-cloudflare-ingest-phase.ts b/apps/web/src/components/infra/cloudflare/use-cloudflare-ingest-phase.ts new file mode 100644 index 000000000..364cfa5de --- /dev/null +++ b/apps/web/src/components/infra/cloudflare/use-cloudflare-ingest-phase.ts @@ -0,0 +1,53 @@ +import { Result, useAtomRefresh, useAtomValue } from "@/lib/effect-atom" +import { retainedQuery } from "@/lib/services/common/atom-client" +import { useIntervalRefresh } from "@/hooks/use-interval-refresh" +import { cloudflareIngestPhase } from "./ingest-phase" + +/** + * Cadence for the self-updating wait. Matched to the poller's own 5-minute tick rather than made + * snappy: the point is that someone who connects and waits sees the page fill in without touching + * anything, not that we ask more often than there is anything new to ask for. + */ +const REFRESH_MS = 30_000 + +/** + * Connection status, warehouse usage, and the ingest phase the two imply — plus the polling that + * turns "collecting" into real numbers in place. Every Cloudflare surface reads its status through + * this hook rather than the query directly, so they agree on what an empty page means. + * + * `phase` is null when the org has no usable connection; the caller's own not-connected and + * needs-permissions states own that case. + */ +export function useCloudflareIngestPhase() { + // Assigned through these so the value hooks and the refresh hooks address the same atoms. + const statusQuery = retainedQuery("integrations", "cloudflareStatus", { + reactivityKeys: ["cloudflareIntegrationStatus"], + }) + const usageQuery = retainedQuery("integrations", "cloudflareUsage", { + reactivityKeys: ["cloudflareIntegrationUsage"], + }) + const statusResult = useAtomValue(statusQuery) + const usageResult = useAtomValue(usageQuery) + const refreshStatus = useAtomRefresh(statusQuery) + const refreshUsage = useAtomRefresh(usageQuery) + + const status = Result.builder(statusResult) + .onSuccess((value) => value) + .orElse(() => null) + // A failed usage read is not proof of no data — passing null holds the phase at the poller's + // own view instead of reporting an absence the warehouse never confirmed. + const usage = Result.builder(usageResult) + .onSuccess((value) => value) + .orElse(() => null) + + // Recomputed every render (including each poll tick) so the phase advances on its own. + const phase = status?.connected === true ? cloudflareIngestPhase(status, usage, Date.now()) : null + + // Poll only while something is expected to change; a live integration refreshes on the page's + // own time-range controls like everything else. + const settling = phase != null && phase.kind !== "live" + useIntervalRefresh(refreshStatus, { intervalMs: REFRESH_MS, enabled: settling }) + useIntervalRefresh(refreshUsage, { intervalMs: REFRESH_MS, enabled: settling }) + + return { statusResult, usageResult, status, usage, phase } +} diff --git a/apps/web/src/components/integrations/cloudflare-account-card.tsx b/apps/web/src/components/integrations/cloudflare-account-card.tsx index 9660b3fd7..01c201306 100644 --- a/apps/web/src/components/integrations/cloudflare-account-card.tsx +++ b/apps/web/src/components/integrations/cloudflare-account-card.tsx @@ -22,6 +22,8 @@ import { IntegrationEmptyMedia, } from "./integration-empty-state" import { CloudflareStatCards } from "./cloudflare-stat-cards" +import { CloudflareIngestBanner } from "@/components/infra/cloudflare/cloudflare-ingest-status" +import { useCloudflareIngestPhase } from "@/components/infra/cloudflare/use-cloudflare-ingest-phase" import { CloudflareWorkersCard, CloudflareZoneBoard, @@ -94,18 +96,10 @@ function CloudflareAccountsStrip({ accounts }: { readonly accounts: ReadonlyArra * Logpush jobs) instead of the manual copy-paste setup. */ export function CloudflareAccountCard() { - // Assigned once so the refresh hooks target the same memoized query atoms. - const statusQuery = retainedQuery("integrations", "cloudflareStatus", { - reactivityKeys: ["cloudflareIntegrationStatus"], - }) - const statusResult = useAtomValue(statusQuery) - - // Warehouse-derived ingest volume: loads independently so the card renders instantly - // from status and the usage columns hydrate (or silently stay absent) afterwards. - const usageQuery = retainedQuery("integrations", "cloudflareUsage", { - reactivityKeys: ["cloudflareIntegrationUsage"], - }) - const usageResult = useAtomValue(usageQuery) + // Status plus the warehouse-derived ingest volume (which loads independently, so the card + // renders instantly from status and the usage columns hydrate afterwards) — and the ingest + // phase they imply, which also drives the polling that fills a fresh connection in place. + const { statusResult, usageResult, phase } = useCloudflareIngestPhase() // Connect flow (popup, busy, refresh-on-return) lives in IntegrationConnectProvider — // shared with the drill-in header's Connect/Reconnect/Disconnect buttons. @@ -339,6 +333,8 @@ export function CloudflareAccountCard() { return (
+ {/* A broken grant or a paused account outranks "still collecting" — never stack both. */} + {banner == null && phase != null ? : null} {hasReadout ? ( <> {!usageFailed ? ( @@ -365,12 +361,15 @@ export function CloudflareAccountCard() {
) : ( + // Nothing discovered yet. The phase banner above already says what is happening and + // when to expect data; this only has to keep the card from looking finished. <> {banner} -

- Traffic data starts arriving within a few minutes — your zones and Workers will appear - here. -

+ {banner != null || phase == null ? ( +

+ Your zones and Workers will appear here once collection starts. +

+ ) : null} )} diff --git a/apps/web/src/routes/infra/cloudflare/$zoneName.tsx b/apps/web/src/routes/infra/cloudflare/$zoneName.tsx index 57d1a6314..4fba22a4e 100644 --- a/apps/web/src/routes/infra/cloudflare/$zoneName.tsx +++ b/apps/web/src/routes/infra/cloudflare/$zoneName.tsx @@ -12,6 +12,11 @@ import { HeroChip, PageHero } from "@/components/infra/primitives/page-hero" import { StatRail, StatRailItem, StatRailLoading } from "@/components/infra/primitives/stat-rail" import { formatBytes, formatPercent } from "@maple/ui/lib/format" import { CloudflareBreakdownPanel } from "@/components/infra/cloudflare/cloudflare-breakdown-panel" +import { + CloudflareIngestEmpty, + CloudflareStalledAction, +} from "@/components/infra/cloudflare/cloudflare-ingest-status" +import { useCloudflareIngestPhase } from "@/components/infra/cloudflare/use-cloudflare-ingest-phase" import { CloudflareEdgeShareBand } from "@/components/infra/cloudflare/cloudflare-edge-share-band" import { CloudflareFilterChips } from "@/components/infra/cloudflare/cloudflare-filter-chips" import { CloudflareFilterSidebarView } from "@/components/infra/cloudflare/cloudflare-filter-sidebar" @@ -204,6 +209,9 @@ function ZoneDetailContent({ onToggleFilter: (key: CloudflareFilterKey, value: string) => void }) { const bucketSeconds = chartBucketSeconds(startTime, endTime) + // A zone drilled into before any data has been collected would otherwise read as "this zone + // has no traffic", which is a different — and wrong — thing to tell someone. + const { phase } = useCloudflareIngestPhase() const detailResult = useRefreshableAtomValue( cloudflareZoneDetailResultAtom({ @@ -233,6 +241,13 @@ function ZoneDetailContent({ .onError((err) => ) .onSuccess((detail, result) => { if (detail.statusBuckets.length === 0 && !result.waiting) { + if (phase != null && phase.kind !== "live" && phase.kind !== "backfilling") { + return ( + + {phase.kind === "stalled" ? : null} + + ) + } return ( diff --git a/apps/web/src/routes/infra/cloudflare/index.tsx b/apps/web/src/routes/infra/cloudflare/index.tsx index d956e79ff..e05af6ca7 100644 --- a/apps/web/src/routes/infra/cloudflare/index.tsx +++ b/apps/web/src/routes/infra/cloudflare/index.tsx @@ -1,7 +1,7 @@ import { useDeferredValue, useMemo, useState, type ReactNode } from "react" import { Link, createFileRoute, useNavigate } from "@tanstack/react-router" import { Schema } from "effect" -import { Result, useAtomValue } from "@/lib/effect-atom" +import { Result } from "@/lib/effect-atom" import { Empty, EmptyDescription, EmptyHeader, EmptyMedia, EmptyTitle } from "@maple/ui/components/ui/empty" import { Skeleton } from "@maple/ui/components/ui/skeleton" @@ -15,6 +15,11 @@ import { CloudflareKpiCards, CloudflareKpiCardsLoading, } from "@/components/infra/cloudflare/cloudflare-kpi-cards" +import { + CloudflareIngestBanner, + CloudflareIngestEmpty, + CloudflareStalledAction, +} from "@/components/infra/cloudflare/cloudflare-ingest-status" import { CloudflareNotConnected } from "@/components/infra/cloudflare/cloudflare-not-connected" import { CloudflarePlatformSection } from "@/components/infra/cloudflare/cloudflare-platform-table" import { @@ -34,7 +39,8 @@ import { cloudflareZonesResultAtom, cloudflareZoneTimeseriesResultAtom, } from "@/lib/services/atoms/warehouse-query-atoms" -import { retainedQuery } from "@/lib/services/common/atom-client" +import { useCloudflareIngestPhase } from "@/components/infra/cloudflare/use-cloudflare-ingest-phase" +import type { CloudflareIngestPhase } from "@/components/infra/cloudflare/ingest-phase" import { useEffectiveTimeRange } from "@/hooks/use-effective-time-range" import { useRefreshableAtomValue } from "@/hooks/use-refreshable-atom-value" import { TimeRangeSearchFields, applyTimeRangeSearch } from "@/components/time-range-picker/search" @@ -71,12 +77,9 @@ function CloudflarePage() { } // Integration-gated (not infra-agent-gated): the page is useful exactly when - // the org has the Cloudflare integration connected with analytics scopes. - const statusResult = useAtomValue( - retainedQuery("integrations", "cloudflareStatus", { - reactivityKeys: ["cloudflareIntegrationStatus"], - }), - ) + // the org has the Cloudflare integration connected with analytics scopes. The hook also + // carries the ingest phase, and polls while a fresh connection is still filling up. + const { statusResult, phase } = useCloudflareIngestPhase() return ( @@ -116,7 +119,13 @@ function CloudflarePage() { if (!status.analyticsCapable) { return } - return + return ( + + ) }) .render()} @@ -128,7 +137,15 @@ function CloudflarePage() { ) } -function CloudflareData({ startTime, endTime }: { startTime: string; endTime: string }) { +function CloudflareData({ + startTime, + endTime, + phase, +}: { + startTime: string + endTime: string + phase: CloudflareIngestPhase | null +}) { const bucketSeconds = chartBucketSeconds(startTime, endTime) // Retained so a manual refresh or a time-range nudge fades the current numbers instead of @@ -176,6 +193,16 @@ function CloudflareData({ startTime, endTime }: { startTime: string; endTime: st .orElse(() => false) if (zonesEmpty && workersEmpty) { + // Two different empties wear the same face otherwise: a connection that has never + // produced anything (say why, and when to expect it) versus a live one whose selected + // window happens to be quiet (say that, and offer the fix — a wider window). + if (phase != null && phase.kind !== "live" && phase.kind !== "backfilling") { + return ( + + {phase.kind === "stalled" ? : null} + + ) + } return ( @@ -184,8 +211,8 @@ function CloudflareData({ startTime, endTime }: { startTime: string; endTime: st No Cloudflare traffic in this window - Analytics ingest in 5-minute batches shortly after the integration connects. Widen the - time range or check back in a few minutes. + This zone set reported no requests over the selected range. Widen the time range, or + check back once more traffic has been collected. @@ -194,6 +221,7 @@ function CloudflareData({ startTime, endTime }: { startTime: string; endTime: st return (
+ {phase == null ? null : } {Result.builder(zonesResult) .onInitial(() => (
diff --git a/packages/domain/src/http/integrations.ts b/packages/domain/src/http/integrations.ts index 731bdac16..c3742362a 100644 --- a/packages/domain/src/http/integrations.ts +++ b/packages/domain/src/http/integrations.ts @@ -88,6 +88,12 @@ export class CloudflareAnalyticsZoneStatus extends Schema.Class