diff --git a/.changeset/readyz-endpoint.md b/.changeset/readyz-endpoint.md new file mode 100644 index 00000000..4ad75052 --- /dev/null +++ b/.changeset/readyz-endpoint.md @@ -0,0 +1,7 @@ +--- +"nostream": minor +--- + +feat(ops): add /readyz readiness probe for Postgres and Redis + +Adds a public readiness endpoint for zero-downtime deploy workflows. HAProxy (or similar) can use `/readyz` to confirm an instance can serve traffic before cutover, while `/healthz` remains a lightweight liveness check. diff --git a/deploy/README.md b/deploy/README.md index 7b27b919..937878db 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -98,8 +98,27 @@ Or use the admin API/UI once `admin.enabled` is configured. ```bash docker compose ps curl -s -H 'Accept: application/nostr+json' http://127.0.0.1:8008/ +curl -s http://127.0.0.1:8008/readyz ``` +## Health checks + +Use the relay HTTP port (default `8008`) for deploy and load-balancer probes: + +| Endpoint | Type | Behavior | Typical use | +|------------|------------|----------------------------------------------------------|-------------------------------| +| `/healthz` | Liveness | Always `200 OK` if the process is running | Restart unhealthy containers | +| `/readyz` | Readiness | `200` when Postgres and Redis respond; `503` otherwise | HAProxy blue/green cutover | + +`/readyz` is unauthenticated and intended for infrastructure. It reuses the same +Postgres and Redis checks as `/admin/health` without requiring admin auth. +Each dependency ping uses the default 3s timeout (`ADMIN_DEPENDENCY_PING_TIMEOUT_MS`). +Set your load balancer check timeout above that (for example HAProxy +`timeout check 5s`) so slow-but-healthy backends do not flap during probes. +Responses are cached in-process for 1s to absorb polling without hammering the DB pool. +Use readiness before routing traffic to a new instance during deploys; graceful +WebSocket draining on shutdown is planned as a follow-up. + ## Image delivery on restricted networks Some hosts cannot reach GHCR over IPv4: diff --git a/src/handlers/request-handlers/get-readyz-request-handler.ts b/src/handlers/request-handlers/get-readyz-request-handler.ts new file mode 100644 index 00000000..0ff9ff1e --- /dev/null +++ b/src/handlers/request-handlers/get-readyz-request-handler.ts @@ -0,0 +1,88 @@ +import { NextFunction, Request, Response } from 'express' + +import { AdminDependencyHealth, collectAdminHealthSnapshot } from '../../utils/admin-health' + +// Public readiness probe for load balancers (e.g. HAProxy blue/green). Unlike /healthz +// (liveness), /readyz returns non-200 when Postgres or Redis is unavailable. +const READY_SNAPSHOT_CACHE_TTL_MS = 1000 + +export interface ReadyzSnapshot { + status: 'ok' | 'unavailable' + database: AdminDependencyHealth + redis: AdminDependencyHealth +} + +interface CachedReadyzSnapshot { + snapshot: ReadyzSnapshot + expiresAt: number +} + +let cachedReadyzSnapshot: CachedReadyzSnapshot | undefined +let inFlightReadyzSnapshot: Promise | undefined + +export const resetReadyzSnapshotCache = (): void => { + cachedReadyzSnapshot = undefined + inFlightReadyzSnapshot = undefined +} + +export const buildReadyzSnapshot = (database: AdminDependencyHealth, redis: AdminDependencyHealth): ReadyzSnapshot => { + const ready = database.ok && redis.ok + + return { + status: ready ? 'ok' : 'unavailable', + database, + redis, + } +} + +const collectReadyzSnapshot = async (): Promise => { + const now = Date.now() + if (cachedReadyzSnapshot && cachedReadyzSnapshot.expiresAt > now) { + return cachedReadyzSnapshot.snapshot + } + + if (inFlightReadyzSnapshot) { + return inFlightReadyzSnapshot + } + + inFlightReadyzSnapshot = (async () => { + const health = await collectAdminHealthSnapshot() + const snapshot = buildReadyzSnapshot(health.database, health.redis) + cachedReadyzSnapshot = { + snapshot, + expiresAt: Date.now() + READY_SNAPSHOT_CACHE_TTL_MS, + } + + return snapshot + })() + + try { + return await inFlightReadyzSnapshot + } finally { + inFlightReadyzSnapshot = undefined + } +} + +const sendReadyzResponse = (res: Response, statusCode: number, snapshot: ReadyzSnapshot): void => { + res + .status(statusCode) + .setHeader('content-type', 'application/json; charset=utf-8') + .setHeader('cache-control', 'no-store') + .send(snapshot) +} + +export const getReadyzRequestHandler = async (_req: Request, res: Response, next: NextFunction) => { + try { + const snapshot = await collectReadyzSnapshot() + const statusCode = snapshot.status === 'ok' ? 200 : 503 + sendReadyzResponse(res, statusCode, snapshot) + } catch { + sendReadyzResponse(res, 503, { + status: 'unavailable', + database: { ok: false }, + redis: { ok: false }, + }) + } + + next() +} diff --git a/src/routes/index.ts b/src/routes/index.ts index d6d91002..1c6f9a0b 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -6,6 +6,7 @@ import admissionRouter from './admissions' import callbacksRouter from './callbacks' import { getHealthRequestHandler } from '../handlers/request-handlers/get-health-request-handler' import { getPrivacyRequestHandler } from '../handlers/request-handlers/get-privacy-request-handler' +import { getReadyzRequestHandler } from '../handlers/request-handlers/get-readyz-request-handler' import { getTermsRequestHandler } from '../handlers/request-handlers/get-terms-request-handler' import invoiceRouter from './invoices' import { rateLimiterMiddleware } from '../handlers/request-handlers/rate-limiter-middleware' @@ -24,9 +25,13 @@ router.use((req, res, next) => { // codeql[js/missing-rate-limiting] router.get('/', rootRequestHandler) +// Liveness: process is running (always 200). Used for "is the container up?" checks. router.get('/healthz', getHealthRequestHandler) -router.get('/terms', getTermsRequestHandler) router.get('/privacy', getPrivacyRequestHandler) +// Readiness: Postgres + Redis must respond. Used before routing traffic during deploys. +// codeql[js/missing-rate-limiting] +router.get('/readyz', getReadyzRequestHandler) +router.get('/terms', getTermsRequestHandler) router.get('/.well-known/nodeinfo', nodeinfoHandler) router.get('/nodeinfo/2.1', nodeinfo21Handler) diff --git a/test/integration/features/response-types/response-types.feature b/test/integration/features/response-types/response-types.feature index 30476579..f646aed5 100644 --- a/test/integration/features/response-types/response-types.feature +++ b/test/integration/features/response-types/response-types.feature @@ -10,6 +10,7 @@ Feature: HTTP response types | / | application/nostr+json | 200 | application/nostr+json | | / | text/html | 200 | text/html | | /healthz | */* | 200 | text/plain | + | /readyz | */* | 200 | application/json | | /terms | */* | 200 | text/html | | /.well-known/nodeinfo | */* | 200 | application/json | | /nodeinfo/2.1 | */* | 200 | application/json | diff --git a/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts b/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts new file mode 100644 index 00000000..25912fd6 --- /dev/null +++ b/test/unit/handlers/request-handlers/get-readyz-request-handler.spec.ts @@ -0,0 +1,153 @@ +import chai from 'chai' +import sinon from 'sinon' +import sinonChai from 'sinon-chai' + +import * as adminHealth from '../../../../src/utils/admin-health' +import { + buildReadyzSnapshot, + getReadyzRequestHandler, + resetReadyzSnapshotCache, +} from '../../../../src/handlers/request-handlers/get-readyz-request-handler' + +chai.use(sinonChai) +const { expect } = chai + +describe('buildReadyzSnapshot', () => { + it('returns ok when database and redis are healthy', () => { + expect(buildReadyzSnapshot({ ok: true }, { ok: true })).to.deep.equal({ + status: 'ok', + database: { ok: true }, + redis: { ok: true }, + }) + }) + + it('returns unavailable when either dependency is unhealthy', () => { + expect(buildReadyzSnapshot({ ok: false }, { ok: true })).to.deep.equal({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: true }, + }) + }) +}) + +describe('getReadyzRequestHandler', () => { + let sandbox: sinon.SinonSandbox + let collectAdminHealthSnapshotStub: sinon.SinonStub + + const healthyAdminSnapshot = { + status: 'ok' as const, + uptimeSeconds: 42, + worker: { type: 'primary' }, + database: { ok: true }, + redis: { ok: true }, + } + + const createResponse = () => + ({ + status: sinon.stub().returnsThis(), + setHeader: sinon.stub().returnsThis(), + send: sinon.stub().returnsThis(), + }) as any + + beforeEach(() => { + sandbox = sinon.createSandbox() + collectAdminHealthSnapshotStub = sandbox.stub(adminHealth, 'collectAdminHealthSnapshot') + resetReadyzSnapshotCache() + }) + + afterEach(() => { + sandbox.restore() + resetReadyzSnapshotCache() + }) + + it('responds with 200 JSON when dependencies are ready', async () => { + collectAdminHealthSnapshotStub.resolves(healthyAdminSnapshot) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(200) + expect(res.setHeader).to.have.been.calledWith('content-type', 'application/json; charset=utf-8') + expect(res.setHeader).to.have.been.calledWith('cache-control', 'no-store') + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'ok', + database: { ok: true }, + redis: { ok: true }, + }) + expect(next).to.have.been.calledOnce + }) + + it('responds with 503 JSON when redis is unavailable', async () => { + collectAdminHealthSnapshotStub.resolves({ + ...healthyAdminSnapshot, + status: 'degraded', + redis: { ok: false }, + }) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'unavailable', + database: { ok: true }, + redis: { ok: false }, + }) + expect(next).to.have.been.calledOnce + }) + + it('responds with 503 JSON when database is unavailable', async () => { + collectAdminHealthSnapshotStub.resolves({ + ...healthyAdminSnapshot, + status: 'degraded', + database: { ok: false }, + }) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: true }, + }) + expect(next).to.have.been.calledOnce + }) + + it('responds with 503 JSON when dependency collection throws', async () => { + collectAdminHealthSnapshotStub.rejects(new Error('boom')) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + + expect(res.status).to.have.been.calledOnceWithExactly(503) + expect(res.setHeader).to.have.been.calledWith('cache-control', 'no-store') + expect(res.send).to.have.been.calledOnceWithExactly({ + status: 'unavailable', + database: { ok: false }, + redis: { ok: false }, + }) + expect(next).to.have.been.calledOnce + }) + + it('reuses a cached snapshot within the 1s TTL', async () => { + collectAdminHealthSnapshotStub.resolves(healthyAdminSnapshot) + + const res = createResponse() + const next = sinon.stub() + + await getReadyzRequestHandler({} as any, res, next) + await getReadyzRequestHandler({} as any, res, next) + + expect(collectAdminHealthSnapshotStub).to.have.been.calledOnce + }) +})