Kubernetes deployment, the mutation-testing gaps, and develop merged in

.claude/skills/cloudflare-tunnel/.env.example

@@ -0,0 +1,64 @@
+# Cloudflare credentials for NAMED tunnels — persistent hostnames on a zone you
+# own, instead of a quick tunnel's throwaway *.trycloudflare.com.
+#
+# Copy to `.env` BESIDE THIS FILE and fill it in. That path is gitignored by the
+# repo-wide `.env` rule, and every script that touches it asks
+# `git check-ignore` first and REFUSES to read or write a token that git could
+# take. Verify yourself before you paste anything:
+#
+#     git check-ignore -v .claude/skills/cloudflare-tunnel/.env
+#
+# Nothing here is required. With no `.env`, every rig falls back to a quick
+# tunnel exactly as before — named mode is an addition, not a replacement.
+#
+# ⚠ THE TOKEN SCOPES TO A ZONE, NOT TO A HOSTNAME. There is no "only these three
+#   subdomains" grant in Cloudflare. A token that can edit DNS in your zone can
+#   edit ANY record in it. Use a zone you are willing to hand to a dev script.
+#   SKILL.md → "Named tunnels" has the exact minting steps and what each
+#   permission is for.
+
+# API token. Minted at
+#   Cloudflare dashboard → My Profile → API Tokens → Create Token → Custom token
+# with exactly two permissions:
+#   Zone    → DNS               → Edit   (Zone Resources: Include → your zone)
+#   Account → Cloudflare Tunnel → Edit   (Account Resources: your account)
+CLOUDFLARE_API_TOKEN=
+
+# The zone the hostnames below live in. The account id is read OUT of the zone
+# record, which is why no account-read permission is needed.
+CLOUDFLARE_ZONE=example.org
+
+# Optional: only if the token can see more than one account and the zone lookup
+# picks the wrong one.
+#CLOUDFLARE_ACCOUNT_ID=
+
+# One hostname per rig; each gets its own tunnel. Leave a line blank or absent
+# and that rig stays on quick tunnels.
+#
+# The app hostname serves the frontend AND Keycloak through caddy's path mux
+# (/realms/*, /resources/* → Keycloak, /objects/* → the object store, everything
+# else → SvelteKit), which is what makes the OIDC redirect work from outside.
+HACKAGON_HOSTNAME=hackagon.example.org
+PLAUSIBLE_HOSTNAME=plausible-hackagon.example.org
+OPENREPLAY_HOSTNAME=openreplay-hackagon.example.org
+
+# The k3d chart rig (.claude/skills/k3d-chart-rig/scripts/tunnel.sh) takes TWO
+# names on ONE tunnel: the helm chart routes the app and Keycloak by HOST, on
+# two separate Ingresses, so there is no single name that reaches both.
+#
+# ⚠ BOTH ARE ONE LABEL DEEP, DELIBERATELY. Cloudflare's free Universal SSL
+# covers the apex and one label — `example.org` and `a.example.org` — and
+# NOTHING below that. `auth.k3d-hackagon.example.org` gets no certificate at the
+# edge and answers a TLS handshake with alert 40, which reads to a browser as a
+# broken site rather than as a missing certificate. Keep them siblings unless
+# the zone has Advanced Certificate Manager.
+K3D_HOSTNAME=k3d-hackagon.example.org
+K3D_AUTH_HOSTNAME=k3d-auth-hackagon.example.org
+
+# Optional: tunnel names as they appear in the Cloudflare dashboard. Defaults
+# below. Change them if two checkouts share one Cloudflare account, or the
+# second one will reuse the first's tunnel and repoint its DNS.
+#HACKAGON_TUNNEL_NAME=hackagon
+#PLAUSIBLE_TUNNEL_NAME=hackagon-plausible
+#OPENREPLAY_TUNNEL_NAME=hackagon-openreplay
+#RIG_TUNNEL_NAME=hackagon-k3d

.claude/skills/cloudflare-tunnel/scripts/auth-wire.sh

@@ -1,9 +1,17 @@
 #!/usr/bin/env bash
-# Rewire OIDC so login works through the quick tunnel's public hostname.
-# Runs INSIDE the dev container (up.sh execs it there).
+# Rewire OIDC so login works through the tunnel's public hostname — quick
+# (*.trycloudflare.com) or NAMED (your own zone). Runs INSIDE the dev container
+# (up.sh execs it there). The hostname is just an argument; nothing here knows
+# or cares which kind it is.
 #
-#   auth-wire.sh <https://X.trycloudflare.com>   wire issuers to the tunnel
-#   auth-wire.sh --restore                       undo (back to localhost)
+#   auth-wire.sh <https://public-host>   wire issuers to that hostname
+#   auth-wire.sh --restore               undo (back to localhost)
+#
+# With a NAMED hostname the second and every later wire is a no-op: the overlay
+# it would write is byte-identical, so nothing is rewritten and — once the
+# running backend confirms it accepts tokens from that issuer — nothing is
+# restarted either. That is the whole reason to prefer a named tunnel: the
+# per-restart re-wiring churn a quick tunnel forces simply does not arise.
 #
 # What wiring does:
 #   1. sanity-check Keycloak answers on the tunnel host with an https issuer
@@ -151,6 +159,30 @@ restart_prod_server() { # <origin>
             "the old issuer — run prod-serve.sh start $origin --no-build" >&2
 }
 
+# Does the RUNNING backend accept a token minted by <issuer-base>?
+#
+# Answers three things, not two, and the third is why this is not a boolean:
+#   0  yes — the running backend validates tokens from that issuer
+#   1  no  — it is running with a different issuer than the one asked about
+#   2  could not ask — Keycloak did not mint a token at all
+#
+# Callers want opposite defaults for that third case, so it is theirs to decide:
+# --restore must not bounce the backend because Keycloak happens to be down,
+# while the wire path must not SKIP a restart on the strength of a question it
+# could not put.
+backend_accepts_issuer() { # <issuer-base-url>
+    local base="${1%/}" token
+    token="$(curl -s --max-time 15 \
+        -X POST "$base/realms/hackagon/protocol/openid-connect/token" \
+        -H "Content-Type: application/x-www-form-urlencoded" \
+        -d client_id=hackagon-backend -d username=alice -d password=aliceandbob \
+        -d grant_type=password -d scope="openid profile" 2>/dev/null |
+        jq -r '.access_token // empty' 2>/dev/null)"
+    [ -n "$token" ] || return 2
+    grpcurl -plaintext -H "authorization: Bearer $token" -max-time 10 \
+        localhost:3000 user.UserService/WhoAmI >/dev/null 2>&1
+}
+
 # Does the RUNNING backend accept a token minted by the LOCALHOST issuer?
 #
 # The overlay says what the configuration intends; this says what the process is
@@ -169,25 +201,17 @@ restart_prod_server() { # <origin>
 # learned: "the server accepted it" and "the server can use it" are different
 # claims, and only one of them can be read off a config file.
 backend_accepts_localhost() {
-    local token
-    token="$(curl -s --max-time 10 \
-        -X POST "http://localhost:8180/realms/hackagon/protocol/openid-connect/token" \
-        -H "Content-Type: application/x-www-form-urlencoded" \
-        -d client_id=hackagon-backend -d username=alice -d password=aliceandbob \
-        -d grant_type=password -d scope="openid profile" 2>/dev/null |
-        jq -r '.access_token // empty' 2>/dev/null)"
-
+    local rc=0
+    backend_accepts_issuer "http://localhost:8180" || rc=$?
     # No token means Keycloak is down or unreachable, which is a different
     # problem: answer "fine" so this never restarts the backend for a reason that
     # has nothing to do with the issuer.
-    if [ -z "$token" ]; then
+    if [ "$rc" = 2 ]; then
         echo "note: could not mint a localhost token (is Keycloak up?) — skipping" \
             "the backend issuer check" >&2
         return 0
     fi
-
-    grpcurl -plaintext -H "authorization: Bearer $token" -max-time 10 \
-        localhost:3000 user.UserService/WhoAmI >/dev/null 2>&1
+    return "$rc"
 }
 
 # Restart the e2e harness's :8081 built server, which reads its OIDC issuer
@@ -276,14 +300,28 @@ esac
 URL_HTTP="http://${URL#https://}"
 HOST="${URL#https://}"
 
-# Fresh trycloudflare hostnames routinely lose the DNS race: the first lookup
-# lands before propagation and the local resolver negative-caches NXDOMAIN,
-# which would break both this script's checks and the frontend's server-side
-# token exchange. Resolve via DNS-over-HTTPS straight at Cloudflare and pin
-# the hostname in /etc/hosts (real edge IP + real TLS cert — traffic still
-# flows through the tunnel). --restore removes the pin.
-if ! getent hosts "$HOST" >/dev/null 2>&1; then
-    echo "==> Local DNS has not caught up with '$HOST' — pinning via /etc/hosts..."
+# Fresh hostnames routinely lose the DNS race: the first lookup lands before
+# propagation and the local resolver negative-caches NXDOMAIN, which would break
+# both this script's checks and the frontend's server-side token exchange.
+# Resolve via DNS-over-HTTPS straight at Cloudflare and pin the hostname in
+# /etc/hosts (real edge IP + real TLS cert — traffic still flows through the
+# tunnel). --restore removes the pin.
+#
+# ⚠ THE TEST IS REACHABILITY, NOT RESOLVABILITY, and the difference is not
+# academic. This used to ask `getent hosts`, which answers YES for a name that
+# resolves to an address nothing here can reach — and that is a real state: the
+# resolver on the machine this was developed against returns Cloudflare's IPv6
+# edge and no A record at all, on a network with no IPv6 route out. Every lookup
+# succeeded, every connection failed instantly, and the pin that exists for
+# exactly this never fired because the question it asked had the wrong answer.
+# A DoH A-record pin fixes both cases, because it forces IPv4.
+host_reachable() {
+    # No -f: any HTTP status means the name resolved AND the edge answered. Only
+    # a resolve (6) or connect (7) failure is what this is looking for.
+    curl -sS -o /dev/null --max-time 8 "https://$HOST/" >/dev/null 2>&1
+}
+if ! host_reachable; then
+    echo "==> '$HOST' is not reachable from here yet — pinning an IPv4 edge via /etc/hosts..."
     ip=$(curl -s --max-time 10 "https://1.1.1.1/dns-query?name=$HOST&type=A" \
         -H "accept: application/dns-json" |
         jq -r '[.Answer[]? | select(.type == 1) | .data][0] // empty')
@@ -332,21 +370,62 @@ echo "==> Pointing frontend/backend issuers at the tunnel (config.local.yaml)...
 # Written through config-overlay.sh, which replaces the `oidc` BLOCK and leaves
 # every other top-level key alone — `replay`, when session replay is wired into
 # the same overlay, is somebody else's and must survive a re-wire.
+CHANGED=0
 write_overlay() { # <path> <yaml-body>
-    bash "$OVERLAY" set "$1" oidc >/dev/null <<EOF
+    local answer
+    answer="$(
+        bash "$OVERLAY" set "$1" oidc <<EOF
 $2
   # Tunnel: $URL — removed by \`auth-wire.sh --restore\`, which down.sh calls
   # for you. A quick-tunnel hostname is this machine's for the next few hours
-  # and belongs nowhere near a tracked file.
+  # and belongs nowhere near a tracked file; a NAMED tunnel's hostname is
+  # stable, and still belongs here rather than in a tracked config, because it
+  # is this machine's deployment choice and not the repo's.
 EOF
+    )"
+    [ "$answer" = "changed" ] && CHANGED=1
+    return 0
 }
 write_overlay "$FRONTEND_LOCAL" "oidc:
   issuer: $URL/realms/$REALM"
 write_overlay "$BACKEND_LOCAL" "oidc:
   issuerurl: \"$URL/realms/$REALM\""
 
-restart_and_wait
-restart_prod_server "$URL"
+# ── the restart, and when it can be skipped ──────────────────────────────────
+#
+# THIS IS WHAT MAKES A NAMED TUNNEL CHEAP TO RE-RUN. With a quick tunnel the
+# hostname is different every time, so the overlay always changes and both
+# processes always have to be restarted — the "re-wiring dance". With a stable
+# hostname the second and every later `up.sh --with-auth` writes byte-identical
+# bytes, and restarting the backend (a cold rebuild here, measured at ~8 min in
+# the devcontainer) to load a config it already has is pure cost.
+#
+# But an unchanged FILE is not a correct PROCESS: a backend started while the
+# overlay said something else is still validating against the issuer it booted
+# with, and that stale-process case is the one this script has already been
+# bitten by three times. So the skip is granted only by the far end — mint a
+# token from the issuer we just wired and see whether the running backend takes
+# it. "Could not ask" restarts, because a skip has to be earned.
+NEEDS_RESTART=1
+if [ "$CHANGED" = 0 ]; then
+    rc=0
+    backend_accepts_issuer "$URL" || rc=$?
+    if [ "$rc" = 0 ]; then
+        NEEDS_RESTART=0
+        echo "==> Issuer already wired to $URL and the running backend accepts its"
+        echo "    tokens — nothing to write, nothing to restart."
+    else
+        echo "==> Issuer overlay was already correct, but the running backend does" \
+            "not accept a token from it (it outlived its config). Restarting:"
+    fi
+fi
+if [ "$NEEDS_RESTART" = 1 ]; then
+    restart_and_wait
+    # Only alongside a real restart: this one rebuilds nothing but does take the
+    # public URL's upstream down and back up, and doing that to load a config it
+    # already holds is a hole in the link for no gain.
+    restart_prod_server "$URL"
+fi
 
 echo
 echo "Login-capable tunnel ready: $URL"

.claude/skills/cloudflare-tunnel/scripts/down.sh

@@ -1,5 +1,14 @@
 #!/usr/bin/env bash
-# Stop all quick tunnels (compose tunnel service + generic port tunnels).
+# Stop all tunnels to this stack — the compose quick tunnel, the named tunnel's
+# container, and any generic port tunnels — and undo the OIDC rewiring.
+#
+# THE NAMED TUNNEL'S HOSTNAME AND DNS RECORD SURVIVE THIS, on purpose. Stopping
+# the container is "take the link down for now"; the whole value of a named
+# tunnel is that the same hostname comes back on the next up.sh, with the issuer
+# wiring still correct. To give the hostname up for good — delete the tunnel and
+# its DNS record from Cloudflare — that is a separate, explicit act:
+#
+#   bash .claude/skills/lib/cf-named-tunnel.sh destroy hackagon <hostname>
 set -euo pipefail
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 ROOT_DIR="$(cd "$HERE/../../../.." && pwd)"
@@ -29,8 +38,18 @@ docker compose -f "$COMPOSE_FILE" exec -T -u vscode -e USER=vscode dev \
     echo "warn: auth restore skipped (dev container not running?)" >&2
 
 docker compose -f "$COMPOSE_FILE" --profile tunnel rm -sf tunnel caddy 2>/dev/null || true
+
+# THIS STACK's named tunnel only. `cf-named-*` also covers the plausible and
+# openreplay rigs, which have their own hostnames and their own down.sh — taking
+# an analytics dashboard offline as a side effect of stopping the app's link is
+# exactly the kind of over-broad cleanup that gets discovered days later.
+docker rm -f "cf-named-${HACKAGON_TUNNEL_NAME:-hackagon}" >/dev/null 2>&1 &&
+    echo "stopped cf-named-${HACKAGON_TUNNEL_NAME:-hackagon}" || true
+
 for name in $(docker ps --format '{{.Names}}' | grep -E '^cf-quicktunnel-' || true); do
     docker rm -f "$name" >/dev/null
     echo "stopped $name"
 done
 echo "tunnels down"
+echo "(a named tunnel's hostname and DNS record are kept — the next up.sh reuses"
+echo " them. Give them up with: lib/cf-named-tunnel.sh destroy <name> <hostname>)"

.claude/skills/cloudflare-tunnel/scripts/prod-serve.sh

@@ -243,8 +243,15 @@ cmd_start() {
     fi
 
     if [ "$build" -eq 1 ]; then
-        echo "==> Building the frontend (pnpm run build:prod)..."
-        (cd "$FRONTEND_DIR" && pnpm run build:prod)
+        # Through the shared serializer, NOT a bare `pnpm build`. This script and
+        # hackathon-e2e/scripts/prod-frontend.sh both build AND SERVE the same
+        # `build/service` tree (this one on :8082, that one on :8081), so they do
+        # not merely race to build it — they race to replace it while the other is
+        # serving it. Two concurrent builds into that one directory corrupted it
+        # three times in one day: `Unexpected end of JSON input`, then a missing
+        # build/service/server/index.js at boot. The helper takes an exclusive lock
+        # and swaps a complete tree into place.
+        bash "$ROOT_DIR/.claude/skills/lib/frontend-build.sh" build
     fi
     if [ ! -f "$FRONTEND_DIR/$SERVER_ENTRY" ]; then
         echo "error: $SERVER_ENTRY missing — run without --no-build." >&2
@@ -387,14 +394,15 @@ cmd_ensure() {
 
 # The bundle is a snapshot of src/, so it has to be rebuilt when src/ moved
 # under it — but rebuilding a current one costs ~40s of a tunnel handover for
-# nothing. Same freshness test as hackathon-e2e/scripts/prod-frontend.sh.
+# nothing.
+#
+# The test lives in the shared builder now, so this script and
+# hackathon-e2e/scripts/prod-frontend.sh cannot drift apart on what "stale"
+# means — they build and serve the SAME build/service tree, and two callers
+# disagreeing about whether it needs rebuilding is one of them rebuilding it
+# under the other.
 bundle_is_stale() {
-    [ -f "$FRONTEND_DIR/$SERVER_ENTRY" ] || return 0
-    local newer
-    newer="$(cd "$FRONTEND_DIR" &&
-        find src static package.json pnpm-lock.yaml svelte.config.js vite.config.ts \
-            -newer "$SERVER_ENTRY" -print -quit 2>/dev/null || true)"
-    [ -n "$newer" ]
+    bash "$ROOT_DIR/.claude/skills/lib/frontend-build.sh" stale
 }
 
 start_with_current_bundle() { # <origin>

.claude/skills/cloudflare-tunnel/scripts/serve-public.sh

@@ -20,15 +20,32 @@
 # Each step is checked, repaired if it can be, and reported. The script ends by
 # driving a REAL login round-trip: serving HTML proves nothing about OIDC.
 #
-# Usage: serve-public.sh [--seed]     (--seed also loads the SDSC archive)
+# Usage: serve-public.sh [--seed] [--with-plausible]
+#   --seed            also load the SDSC archive (six real past editions)
+#   --with-plausible  bring the Plausible rig up on its OWN tunnel and wire the
+#                     app at it. Off by default: it costs ~750 MB idle, and an
+#                     analytics dashboard full of Playwright traffic is worse
+#                     than an empty one — unwire before any suite run.
 set -euo pipefail
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SKILLS="$(cd "$HERE/../.." && pwd)"
 ROOT_DIR="$(cd "$SKILLS/.." && pwd)"
 E2E="$SKILLS/hackathon-e2e"
 
 SEED=0
-[ "${1:-}" = "--seed" ] && SEED=1
+PLAUSIBLE=0
+# Set only when the rig comes up; the Ready block tests it with ${VAR:+…}.
+PLAUSIBLE_URL=""
+for arg in "$@"; do
+    case "$arg" in
+    --seed) SEED=1 ;;
+    --with-plausible) PLAUSIBLE=1 ;;
+    *)
+        echo "unknown argument: $arg (see the usage comment)" >&2
+        exit 2
+        ;;
+    esac
+done
 
 step() {
     echo
@@ -99,6 +116,22 @@ if [ "$SEED" -eq 1 ]; then
     nix "E2E_KEYCLOAK_URL=$URL bash .claude/skills/seed-past-hackathons/scripts/prizes.sh" >/dev/null 2>&1 || true
 fi
 
+# ── 4b. analytics, opt-in ────────────────────────────────────────────────────
+# Its own tunnel, not this one: the tracking script and the dashboard need a
+# public origin of their own, and sharing this hostname would put the app and a
+# third-party dashboard behind one link.
+if [ "$PLAUSIBLE" -eq 1 ]; then
+    step "Plausible"
+    if bash "$SKILLS/plausible-stack/scripts/up.sh" >/dev/null 2>&1; then
+        nix 'bash .claude/skills/plausible-stack/scripts/wire-frontend.sh' >/dev/null 2>&1 &&
+            ok "wired — the frontend loads the tracker" ||
+            warn "rig is up but wiring failed; run plausible-stack/scripts/wire-frontend.sh"
+        PLAUSIBLE_URL="$(bash "$SKILLS/plausible-stack/scripts/url.sh" 2>/dev/null | grep -oE 'https://[a-z0-9-]+\.trycloudflare\.com' | tail -1 || true)"
+    else
+        warn "could not start Plausible — continuing without it"
+    fi
+fi
+
 # ── 5. prove a login ─────────────────────────────────────────────────────────
 # The whole point. Every step above can be green while signing in is broken,
 # and that combination has happened repeatedly: the issuer, the ORIGIN and a
@@ -124,7 +157,12 @@ cat <<EOF
   Password   aliceandbob   (dev only — never a real deployment)
 
   Keycloak admin console  $URL/admin/  (admin / admin)
-
+${PLAUSIBLE_URL:+
+  Analytics  $PLAUSIBLE_URL   (admin@hackagon.test — password in
+             .claude/skills/plausible-stack/.secrets.env)
+             Unwire before a suite run, or the numbers become Playwright:
+               .claude/skills/plausible-stack/scripts/wire-frontend.sh --restore
+}
   Stop the public link:  .claude/skills/cloudflare-tunnel/scripts/down.sh
   The quick-tunnel URL dies with the tunnel; re-run this to mint a new one.
 EOF