From 2a88463ef3d5ecad5d93dc4c95e93c7ee634c992 Mon Sep 17 00:00:00 2001 From: Hrt-Htk Date: Sun, 19 Jul 2026 21:19:22 +0200 Subject: [PATCH] fix(proxy): raise first-byte timeout to 240s to cover large-context prefill (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lifecycle testing exposed a false-positive in the #13 fix: this router withholds response headers until the worker emits its first token, so time-to-headers includes the full prompt prefill. A near-limit cold-cache prefill (131k tokens at ~800 tok/s) takes ~165s; the 60s bound tripped on large contexts and forced needless worker reloads (losing the prompt cache, which can cascade). Raise the default to 240s — above worst-case legitimate prefill — so a genuinely hung worker is still detected while normal large-context requests are not. Still overridable via $PROXY_FIRST_BYTE_TIMEOUT. Co-Authored-By: Claude Opus 4.8 --- proxy_request_handlers.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/proxy_request_handlers.py b/proxy_request_handlers.py index 3a8624f..89c058f 100644 --- a/proxy_request_handlers.py +++ b/proxy_request_handlers.py @@ -25,9 +25,17 @@ # forwarded it to a dead/hung model-worker child, and never responded, while the # proxy awaited with timeout=None and hung forever. A hung worker is # indistinguishable from an infinitely-slow one, so — like nginx's -# proxy_read_timeout — we bound the wait. Generous enough that a cold model load -# (~15-25s) is never mistaken for a hang. Override via $PROXY_FIRST_BYTE_TIMEOUT. -FIRST_BYTE_TIMEOUT = float(os.environ.get("PROXY_FIRST_BYTE_TIMEOUT", "60")) +# proxy_read_timeout — we bound the wait. +# +# CALIBRATION (measured 2026-07-19): this router withholds response headers until +# the worker emits its first token, so time-to-headers includes the full prompt +# *prefill*. A near-limit cold-cache prefill (131k tokens at ~800 tok/s) takes +# ~165s. A 60s bound false-tripped on large contexts and forced needless worker +# reloads (losing the prompt cache, which can cascade). So the bound must sit +# above the worst-case legitimate prefill: 240s covers a full-context cold prefill +# with margin, while a genuinely hung worker is still detected (just later). +# Override via $PROXY_FIRST_BYTE_TIMEOUT. +FIRST_BYTE_TIMEOUT = float(os.environ.get("PROXY_FIRST_BYTE_TIMEOUT", "240")) def _strip_chat_prefix(path: str) -> str: