From a0eb81599b507a5bf575b097acbfca199bcadb80 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Tue, 8 Sep 2026 11:15:55 +0530 Subject: [PATCH 1/2] feat(ops): add /readyz readiness probe for Postgres and Redis Public readiness endpoint for HAProxy blue/green cutover. /healthz stays liveness-only; /readyz returns 503 when Postgres or Redis is unavailable. --- .changeset/readyz-endpoint.md | 7 + deploy/README.md | 15 +++ .../get-readyz-request-handler.ts | 42 ++++++ src/routes/index.ts | 4 + .../response-types/response-types.feature | 1 + .../get-readyz-request-handler.spec.ts | 123 ++++++++++++++++++ 6 files changed, 192 insertions(+) create mode 100644 .changeset/readyz-endpoint.md create mode 100644 src/handlers/request-handlers/get-readyz-request-handler.ts create mode 100644 test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts diff --git a/.changeset/readyz-endpoint.md b/.changeset/readyz-endpoint.md new file mode 100644 index 00000000..d98ea5da --- /dev/null +++ b/.changeset/readyz-endpoint.md @@ -0,0 +1,7 @@ +--- +"nostream": patch +--- + +feat(ops): add /readyz readiness probe for Postgres and Redis + +Adds a public readiness endpoint for zero-downtime deploy workflows. HAProxy (or similar) can use `/readyz` to confirm an instance can serve traffic before cutover, while `/healthz` remains a lightweight liveness check. diff --git a/deploy/README.md b/deploy/README.md index 7b27b919..843cf4e8 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -98,8 +98,23 @@ Or use the admin API/UI once `admin.enabled` is configured. ```bash docker compose ps curl -s -H 'Accept: application/nostr+json' http://127.0.0.1:8008/ +curl -s http://127.0.0.1:8008/readyz ``` +## Health checks + +Use the relay HTTP port (default `8008`) for deploy and load-balancer probes: + +| Endpoint | Type | Behavior | Typical use | +|------------|------------|----------------------------------------------------------|-------------------------------| +| `/healthz` | Liveness | Always `200 OK` if the process is running | Restart unhealthy containers | +| `/readyz` | Readiness | `200` when Postgres and Redis respond; `503` otherwise | HAProxy blue/green cutover | + +`/readyz` is unauthenticated and intended for infrastructure. It reuses the same +Postgres and Redis checks as `/admin/health` without requiring admin auth. +Use readiness before routing traffic to a new instance during deploys; graceful +WebSocket draining on shutdown is planned as a follow-up. + ## Image delivery on restricted networks Some hosts cannot reach GHCR over IPv4: diff --git a/src/handlers/request-handlers/get-readyz-request-handler.ts b/src/handlers/request-handlers/get-readyz-request-handler.ts new file mode 100644 index 00000000..c3b59825 --- /dev/null +++ b/src/handlers/request-handlers/get-readyz-request-handler.ts @@ -0,0 +1,42 @@ +import { NextFunction, Request, Response } from 'express' + +import { AdminDependencyHealth, collectAdminHealthSnapshot } from '../../utils/admin-health' + +// Public readiness probe for load balancers (e.g. HAProxy blue/green). Unlike /healthz +// (liveness), /readyz returns non-200 when Postgres or Redis is unavailable. +export interface ReadyzSnapshot { + status: 'ok' | 'unavailable' + database: AdminDependencyHealth + redis: AdminDependencyHealth +} + +export const buildReadyzSnapshot = (database: AdminDependencyHealth, redis: AdminDependencyHealth): ReadyzSnapshot => { + const ready = database.ok && redis.ok + + return { + status: ready ? 'ok' : 'unavailable', + database, + redis, + } +} + +export const getReadyzRequestHandler = async (_req: Request, res: Response, next: NextFunction) => { + try { + const health = await collectAdminHealthSnapshot() + const snapshot = buildReadyzSnapshot(health.database, health.redis) + const statusCode = snapshot.status === 'ok' ? 200 : 503 + + res.status(statusCode).setHeader('content-type', 'application/json; charset=utf-8').send(snapshot) + } catch { + res + .status(503) + .setHeader('content-type', 'application/json; charset=utf-8') + .send({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: false }, + } satisfies ReadyzSnapshot) + } + + next() +} diff --git a/src/routes/index.ts b/src/routes/index.ts index d6d91002..6341040b 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -5,6 +5,7 @@ import adminRouter from './admin' import admissionRouter from './admissions' import callbacksRouter from './callbacks' import { getHealthRequestHandler } from '../handlers/request-handlers/get-health-request-handler' +import { getReadyzRequestHandler } from '../handlers/request-handlers/get-readyz-request-handler' import { getPrivacyRequestHandler } from '../handlers/request-handlers/get-privacy-request-handler' import { getTermsRequestHandler } from '../handlers/request-handlers/get-terms-request-handler' import invoiceRouter from './invoices' @@ -24,7 +25,10 @@ router.use((req, res, next) => { // codeql[js/missing-rate-limiting] router.get('/', rootRequestHandler) +// Liveness: process is running (always 200). Used for "is the container up?" checks. router.get('/healthz', getHealthRequestHandler) +// Readiness: Postgres + Redis must respond. Used before routing traffic during deploys. +router.get('/readyz', getReadyzRequestHandler) router.get('/terms', getTermsRequestHandler) router.get('/privacy', getPrivacyRequestHandler) diff --git a/test/integration/features/response-types/response-types.feature b/test/integration/features/response-types/response-types.feature index 30476579..f646aed5 100644 --- a/test/integration/features/response-types/response-types.feature +++ b/test/integration/features/response-types/response-types.feature @@ -10,6 +10,7 @@ Feature: HTTP response types | / | application/nostr+json | 200 | application/nostr+json | | / | text/html | 200 | text/html | | /healthz | */* | 200 | text/plain | + | /readyz | */* | 200 | application/json | | /terms | */* | 200 | text/html | | /.well-known/nodeinfo | */* | 200 | application/json | | /nodeinfo/2.1 | */* | 200 | application/json | diff --git a/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts b/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts new file mode 100644 index 00000000..8bf89a7c --- /dev/null +++ b/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts @@ -0,0 +1,123 @@ +import chai from 'chai' +import sinon from 'sinon' +import sinonChai from 'sinon-chai' + +import * as adminHealth from '../../../../src/utils/admin-health' +import { + buildReadyzSnapshot, + getReadyzRequestHandler, +} from '../../../../src/handlers/request-handlers/get-readyz-request-handler' + +chai.use(sinonChai) +const { expect } = chai + +describe('buildReadyzSnapshot', () => { + it('returns ok when database and redis are healthy', () => { + expect(buildReadyzSnapshot({ ok: true }, { ok: true })).to.deep.equal({ + status: 'ok', + database: { ok: true }, + redis: { ok: true }, + }) + }) + + it('returns unavailable when either dependency is unhealthy', () => { + expect(buildReadyzSnapshot({ ok: false }, { ok: true })).to.deep.equal({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: true }, + }) + }) +}) + +describe('getReadyzRequestHandler', () => { + let sandbox: sinon.SinonSandbox + let collectAdminHealthSnapshotStub: sinon.SinonStub + + beforeEach(() => { + sandbox = sinon.createSandbox() + collectAdminHealthSnapshotStub = sandbox.stub(adminHealth, 'collectAdminHealthSnapshot') + }) + + afterEach(() => { + sandbox.restore() + }) + + it('responds with 200 JSON when dependencies are ready', async () => { + collectAdminHealthSnapshotStub.resolves({ + status: 'ok', + uptimeSeconds: 42, + worker: { type: 'primary' }, + database: { ok: true }, + redis: { ok: true }, + }) + + const req = {} as any + const res = { + status: sinon.stub().returnsThis(), + setHeader: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + } as any + const next = sinon.stub() + + await getReadyzRequestHandler(req, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(200) + expect(res.setHeader).to.have.been.calledOnceWithExactly('content-type', 'application/json; charset=utf-8') + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'ok', + database: { ok: true }, + redis: { ok: true }, + }) + expect(next).to.have.been.calledOnce + }) + + it('responds with 503 JSON when a dependency is unavailable', async () => { + collectAdminHealthSnapshotStub.resolves({ + status: 'degraded', + uptimeSeconds: 42, + worker: { type: 'primary' }, + database: { ok: true }, + redis: { ok: false }, + }) + + const req = {} as any + const res = { + status: sinon.stub().returnsThis(), + setHeader: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + } as any + const next = sinon.stub() + + await getReadyzRequestHandler(req, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'unavailable', + database: { ok: true }, + redis: { ok: false }, + }) + expect(next).to.have.been.calledOnce + }) + + it('responds with 503 JSON when dependency collection throws', async () => { + collectAdminHealthSnapshotStub.rejects(new Error('boom')) + + const req = {} as any + const res = { + status: sinon.stub().returnsThis(), + setHeader: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + } as any + const next = sinon.stub() + + await getReadyzRequestHandler(req, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: false }, + }) + expect(next).to.have.been.calledOnce + }) +}) From 133184b59e9673857aef18888809ef963b0081df Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Tue, 8 Sep 2026 19:00:39 +0530 Subject: [PATCH 2/2] fix(ops): address /readyz PR review feedback --- .changeset/readyz-endpoint.md | 2 +- deploy/README.md | 4 + .../get-readyz-request-handler.ts | 70 +++++++++++--- src/routes/index.ts | 5 +- .../get-readyz-request-handler.spec.ts | 96 ++++++++++++------- 5 files changed, 129 insertions(+), 48 deletions(-) diff --git a/.changeset/readyz-endpoint.md b/.changeset/readyz-endpoint.md index d98ea5da..4ad75052 100644 --- a/.changeset/readyz-endpoint.md +++ b/.changeset/readyz-endpoint.md @@ -1,5 +1,5 @@ --- -"nostream": patch +"nostream": minor --- feat(ops): add /readyz readiness probe for Postgres and Redis diff --git a/deploy/README.md b/deploy/README.md index 843cf4e8..937878db 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -112,6 +112,10 @@ Use the relay HTTP port (default `8008`) for deploy and load-balancer probes: `/readyz` is unauthenticated and intended for infrastructure. It reuses the same Postgres and Redis checks as `/admin/health` without requiring admin auth. +Each dependency ping uses the default 3s timeout (`ADMIN_DEPENDENCY_PING_TIMEOUT_MS`). +Set your load balancer check timeout above that (for example HAProxy +`timeout check 5s`) so slow-but-healthy backends do not flap during probes. +Responses are cached in-process for 1s to absorb polling without hammering the DB pool. Use readiness before routing traffic to a new instance during deploys; graceful WebSocket draining on shutdown is planned as a follow-up. diff --git a/src/handlers/request-handlers/get-readyz-request-handler.ts b/src/handlers/request-handlers/get-readyz-request-handler.ts index c3b59825..0ff9ff1e 100644 --- a/src/handlers/request-handlers/get-readyz-request-handler.ts +++ b/src/handlers/request-handlers/get-readyz-request-handler.ts @@ -4,12 +4,27 @@ import { AdminDependencyHealth, collectAdminHealthSnapshot } from '../../utils/a // Public readiness probe for load balancers (e.g. HAProxy blue/green). Unlike /healthz // (liveness), /readyz returns non-200 when Postgres or Redis is unavailable. +const READY_SNAPSHOT_CACHE_TTL_MS = 1000 + export interface ReadyzSnapshot { status: 'ok' | 'unavailable' database: AdminDependencyHealth redis: AdminDependencyHealth } +interface CachedReadyzSnapshot { + snapshot: ReadyzSnapshot + expiresAt: number +} + +let cachedReadyzSnapshot: CachedReadyzSnapshot | undefined +let inFlightReadyzSnapshot: Promise | undefined + +export const resetReadyzSnapshotCache = (): void => { + cachedReadyzSnapshot = undefined + inFlightReadyzSnapshot = undefined +} + export const buildReadyzSnapshot = (database: AdminDependencyHealth, redis: AdminDependencyHealth): ReadyzSnapshot => { const ready = database.ok && redis.ok @@ -20,22 +35,53 @@ export const buildReadyzSnapshot = (database: AdminDependencyHealth, redis: Admi } } -export const getReadyzRequestHandler = async (_req: Request, res: Response, next: NextFunction) => { - try { +const collectReadyzSnapshot = async (): Promise => { + const now = Date.now() + if (cachedReadyzSnapshot && cachedReadyzSnapshot.expiresAt > now) { + return cachedReadyzSnapshot.snapshot + } + + if (inFlightReadyzSnapshot) { + return inFlightReadyzSnapshot + } + + inFlightReadyzSnapshot = (async () => { const health = await collectAdminHealthSnapshot() const snapshot = buildReadyzSnapshot(health.database, health.redis) - const statusCode = snapshot.status === 'ok' ? 200 : 503 + cachedReadyzSnapshot = { + snapshot, + expiresAt: Date.now() + READY_SNAPSHOT_CACHE_TTL_MS, + } + + return snapshot + })() + + try { + return await inFlightReadyzSnapshot + } finally { + inFlightReadyzSnapshot = undefined + } +} - res.status(statusCode).setHeader('content-type', 'application/json; charset=utf-8').send(snapshot) +const sendReadyzResponse = (res: Response, statusCode: number, snapshot: ReadyzSnapshot): void => { + res + .status(statusCode) + .setHeader('content-type', 'application/json; charset=utf-8') + .setHeader('cache-control', 'no-store') + .send(snapshot) +} + +export const getReadyzRequestHandler = async (_req: Request, res: Response, next: NextFunction) => { + try { + const snapshot = await collectReadyzSnapshot() + const statusCode = snapshot.status === 'ok' ? 200 : 503 + sendReadyzResponse(res, statusCode, snapshot) } catch { - res - .status(503) - .setHeader('content-type', 'application/json; charset=utf-8') - .send({ - status: 'unavailable', - database: { ok: false }, - redis: { ok: false }, - } satisfies ReadyzSnapshot) + sendReadyzResponse(res, 503, { + status: 'unavailable', + database: { ok: false }, + redis: { ok: false }, + }) } next() diff --git a/src/routes/index.ts b/src/routes/index.ts index 6341040b..1c6f9a0b 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -5,8 +5,8 @@ import adminRouter from './admin' import admissionRouter from './admissions' import callbacksRouter from './callbacks' import { getHealthRequestHandler } from '../handlers/request-handlers/get-health-request-handler' -import { getReadyzRequestHandler } from '../handlers/request-handlers/get-readyz-request-handler' import { getPrivacyRequestHandler } from '../handlers/request-handlers/get-privacy-request-handler' +import { getReadyzRequestHandler } from '../handlers/request-handlers/get-readyz-request-handler' import { getTermsRequestHandler } from '../handlers/request-handlers/get-terms-request-handler' import invoiceRouter from './invoices' import { rateLimiterMiddleware } from '../handlers/request-handlers/rate-limiter-middleware' @@ -27,10 +27,11 @@ router.use((req, res, next) => { router.get('/', rootRequestHandler) // Liveness: process is running (always 200). Used for "is the container up?" checks. router.get('/healthz', getHealthRequestHandler) +router.get('/privacy', getPrivacyRequestHandler) // Readiness: Postgres + Redis must respond. Used before routing traffic during deploys. +// codeql[js/missing-rate-limiting] router.get('/readyz', getReadyzRequestHandler) router.get('/terms', getTermsRequestHandler) -router.get('/privacy', getPrivacyRequestHandler) router.get('/.well-known/nodeinfo', nodeinfoHandler) router.get('/nodeinfo/2.1', nodeinfo21Handler) diff --git a/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts b/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts index 8bf89a7c..25912fd6 100644 --- a/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts +++ b/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts @@ -6,6 +6,7 @@ import * as adminHealth from '../../../../src/utils/admin-health' import { buildReadyzSnapshot, getReadyzRequestHandler, + resetReadyzSnapshotCache, } from '../../../../src/handlers/request-handlers/get-readyz-request-handler' chai.use(sinonChai) @@ -33,36 +34,43 @@ describe('getReadyzRequestHandler', () => { let sandbox: sinon.SinonSandbox let collectAdminHealthSnapshotStub: sinon.SinonStub + const healthyAdminSnapshot = { + status: 'ok' as const, + uptimeSeconds: 42, + worker: { type: 'primary' }, + database: { ok: true }, + redis: { ok: true }, + } + + const createResponse = () => + ({ + status: sinon.stub().returnsThis(), + setHeader: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }) as any + beforeEach(() => { sandbox = sinon.createSandbox() collectAdminHealthSnapshotStub = sandbox.stub(adminHealth, 'collectAdminHealthSnapshot') + resetReadyzSnapshotCache() }) afterEach(() => { sandbox.restore() + resetReadyzSnapshotCache() }) it('responds with 200 JSON when dependencies are ready', async () => { - collectAdminHealthSnapshotStub.resolves({ - status: 'ok', - uptimeSeconds: 42, - worker: { type: 'primary' }, - database: { ok: true }, - redis: { ok: true }, - }) + collectAdminHealthSnapshotStub.resolves(healthyAdminSnapshot) - const req = {} as any - const res = { - status: sinon.stub().returnsThis(), - setHeader: sinon.stub().returnsThis(), - send: sinon.stub().returnsThis(), - } as any + const res = createResponse() const next = sinon.stub() - await getReadyzRequestHandler(req, res, next) + await getReadyzRequestHandler({} as any, res, next) expect(res.status).to.have.been.calledOnceWithExactly(200) - expect(res.setHeader).to.have.been.calledOnceWithExactly('content-type', 'application/json; charset=utf-8') + expect(res.setHeader).to.have.been.calledWith('content-type', 'application/json; charset=utf-8') + expect(res.setHeader).to.have.been.calledWith('cache-control', 'no-store') expect(res.send).to.have.been.calledOnceWithExactly({ status: 'ok', database: { ok: true }, @@ -71,24 +79,17 @@ describe('getReadyzRequestHandler', () => { expect(next).to.have.been.calledOnce }) - it('responds with 503 JSON when a dependency is unavailable', async () => { + it('responds with 503 JSON when redis is unavailable', async () => { collectAdminHealthSnapshotStub.resolves({ + ...healthyAdminSnapshot, status: 'degraded', - uptimeSeconds: 42, - worker: { type: 'primary' }, - database: { ok: true }, redis: { ok: false }, }) - const req = {} as any - const res = { - status: sinon.stub().returnsThis(), - setHeader: sinon.stub().returnsThis(), - send: sinon.stub().returnsThis(), - } as any + const res = createResponse() const next = sinon.stub() - await getReadyzRequestHandler(req, res, next) + await getReadyzRequestHandler({} as any, res, next) expect(res.status).to.have.been.calledOnceWithExactly(503) expect(res.send).to.have.been.calledOnceWithExactly({ @@ -99,20 +100,37 @@ describe('getReadyzRequestHandler', () => { expect(next).to.have.been.calledOnce }) + it('responds with 503 JSON when database is unavailable', async () => { + collectAdminHealthSnapshotStub.resolves({ + ...healthyAdminSnapshot, + status: 'degraded', + database: { ok: false }, + }) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: true }, + }) + expect(next).to.have.been.calledOnce + }) + it('responds with 503 JSON when dependency collection throws', async () => { collectAdminHealthSnapshotStub.rejects(new Error('boom')) - const req = {} as any - const res = { - status: sinon.stub().returnsThis(), - setHeader: sinon.stub().returnsThis(), - send: sinon.stub().returnsThis(), - } as any + const res = createResponse() const next = sinon.stub() - await getReadyzRequestHandler(req, res, next) + await getReadyzRequestHandler({} as any, res, next) expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.setHeader).to.have.been.calledWith('cache-control', 'no-store') expect(res.send).to.have.been.calledOnceWithExactly({ status: 'unavailable', database: { ok: false }, @@ -120,4 +138,16 @@ describe('getReadyzRequestHandler', () => { }) expect(next).to.have.been.calledOnce }) + + it('reuses a cached snapshot within the 1s TTL', async () => { + collectAdminHealthSnapshotStub.resolves(healthyAdminSnapshot) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + await getReadyzRequestHandler({} as any, res, next) + + expect(collectAdminHealthSnapshotStub).to.have.been.calledOnce + }) })