QuantStrategyLab · Pigbibi · Aug 27, 2026 · Aug 27, 2026
@@ -21,6 +21,8 @@ jobs:
           set -euo pipefail
           bash tests/test_install_2fa_bot_watcher.sh
           bash tests/test_wait_for_ib_gateway_ready.sh
+          bash tests/test_gateway_recovery_scripts.sh
+          bash tests/test_monitor_ib_gateway_ready.sh
           bash tests/test_workflow_shared_config.sh
           bash tests/test_docker_compose_ports.sh
 

@@ -10,7 +10,14 @@ compose_service_name="${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}"
 unit_suffix="${IB_GATEWAY_UNIT_SUFFIX:-}"
 gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}"
 compose_file="${COMPOSE_FILE:-docker-compose.yml}"
-health_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300}"
+healthcheck_tick_seconds="${IB_GATEWAY_HEALTHCHECK_TICK_SECONDS:-300}"
+health_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900}"
+execution_window_interval_seconds="${IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300}"
+execution_window_times="${IB_GATEWAY_EXECUTION_WINDOW_TIMES:-09:45,15:45}"
+execution_window_minutes="${IB_GATEWAY_EXECUTION_WINDOW_MINUTES:-60}"
+execution_window_timezone="${IB_GATEWAY_EXECUTION_WINDOW_TIMEZONE:-America/New_York}"
+failure_threshold="${IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2}"
+probe_timeout_seconds="${IB_GATEWAY_HEALTHCHECK_PROBE_TIMEOUT_SECONDS:-30}"
 daily_restart_calendar="${IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC}"
 
 . "$script_dir/ibkr_gateway_units.sh"
@@ -22,12 +29,18 @@ else
   default_lock_file="/var/lock/ib_gateway_recovery.lock"
 fi
 recovery_lock_file="${IB_GATEWAY_RECOVERY_LOCK_FILE:-$default_lock_file}"
+if [ -n "$resolved_unit_suffix" ]; then
+  default_state_file="/var/lib/ib_gateway_healthcheck/${resolved_unit_suffix}.state"
+else
+  default_state_file="/var/lib/ib_gateway_healthcheck/default.state"
+fi
+state_file="${IB_GATEWAY_HEALTHCHECK_STATE_FILE:-$default_state_file}"
 
 install -d "$systemd_dir"
 
 cat >"$systemd_dir/$IBKR_GATEWAY_HEALTHCHECK_SERVICE" <<EOF
 [Unit]
-Description=Check and recover IBKR Gateway API readiness
+Description=Adaptively check and recover IBKR Gateway API readiness
 After=docker.service network-online.target
 Wants=docker.service network-online.target
 
@@ -40,16 +53,24 @@ Environment=IB_GATEWAY_MODE=$gateway_mode
 Environment=COMPOSE_FILE=$compose_file
 Environment=IB_GATEWAY_RECOVERY_LOCK_FILE=$recovery_lock_file
 Environment=IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS=0
-ExecStart=/bin/bash -lc 'cd "$repo_dir" && exec ./scripts/recover_ib_gateway_ready.sh "$gateway_mode"'
+Environment=IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=$health_interval_seconds
+Environment=IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS=$execution_window_interval_seconds
+Environment=IB_GATEWAY_EXECUTION_WINDOW_TIMES=$execution_window_times
+Environment=IB_GATEWAY_EXECUTION_WINDOW_MINUTES=$execution_window_minutes
+Environment=IB_GATEWAY_EXECUTION_WINDOW_TIMEZONE=$execution_window_timezone
+Environment=IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=$failure_threshold
+Environment=IB_GATEWAY_HEALTHCHECK_PROBE_TIMEOUT_SECONDS=$probe_timeout_seconds
+Environment=IB_GATEWAY_HEALTHCHECK_STATE_FILE=$state_file
+ExecStart=/bin/bash -lc 'cd "$repo_dir" && exec ./scripts/monitor_ib_gateway_ready.sh "$gateway_mode"'
 EOF
 
 cat >"$systemd_dir/$IBKR_GATEWAY_HEALTHCHECK_TIMER" <<EOF
 [Unit]
-Description=Run IBKR Gateway API readiness recovery every $health_interval_seconds seconds
+Description=Wake the adaptive IBKR Gateway API readiness monitor every $healthcheck_tick_seconds seconds
 
 [Timer]
-OnActiveSec=$health_interval_seconds
-OnUnitActiveSec=$health_interval_seconds
+OnActiveSec=$healthcheck_tick_seconds
+OnUnitActiveSec=$healthcheck_tick_seconds
 Unit=$IBKR_GATEWAY_HEALTHCHECK_SERVICE
 
 [Install]

@@ -0,0 +1,184 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Run a cheap, read-only Gateway readiness probe on an adaptive cadence.  The
+# systemd timer wakes every few minutes, but this script only opens an IB API
+# session every 15 minutes outside configured execution windows.  Two
+# consecutive failed probes are required before the existing recovery script
+# is allowed to restart or recreate the Gateway container.
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}"
+normal_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900}"
+execution_window_interval_seconds="${IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300}"
+execution_window_times="${IB_GATEWAY_EXECUTION_WINDOW_TIMES:-09:45,15:45}"
+execution_window_minutes="${IB_GATEWAY_EXECUTION_WINDOW_MINUTES:-60}"
+execution_window_timezone="${IB_GATEWAY_EXECUTION_WINDOW_TIMEZONE:-America/New_York}"
+failure_threshold="${IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2}"
+probe_timeout_seconds="${IB_GATEWAY_HEALTHCHECK_PROBE_TIMEOUT_SECONDS:-30}"
+state_file="${IB_GATEWAY_HEALTHCHECK_STATE_FILE:-/var/lib/ib_gateway_healthcheck/default.state}"
+readiness_script="${IB_GATEWAY_READINESS_SCRIPT:-$script_dir/wait_for_ib_gateway_ready.sh}"
+recovery_script="${IB_GATEWAY_RECOVERY_SCRIPT:-$script_dir/recover_ib_gateway_ready.sh}"
+configured_now_epoch="${IB_GATEWAY_HEALTHCHECK_NOW_EPOCH:-}"
+
+is_positive_integer() {
+  [[ "$1" =~ ^[1-9][0-9]*$ ]]
+}
+
+for value_name in \
+  normal_interval_seconds \
+  execution_window_interval_seconds \
+  execution_window_minutes \
+  failure_threshold \
+  probe_timeout_seconds; do
+  value="${!value_name}"
+  if ! is_positive_integer "$value"; then
+    echo "$value_name must be a positive integer." >&2
+    exit 2
+  fi
+done
+
+if [ -n "$configured_now_epoch" ]; then
+  if ! [[ "$configured_now_epoch" =~ ^[0-9]+$ ]]; then
+    echo "IB_GATEWAY_HEALTHCHECK_NOW_EPOCH must be a non-negative integer." >&2
+    exit 2
+  fi
+  now_epoch="$configured_now_epoch"
+else
+  now_epoch="$(date +%s)"
+fi
+
+state_dir="$(dirname "$state_file")"
+install -d -m 700 "$state_dir"
+
+last_check_epoch=0
+failure_count=0
+if [ -f "$state_file" ]; then
+  while IFS='=' read -r key value; do
+    case "$key" in
+      last_check_epoch)
+        if [[ "$value" =~ ^[0-9]+$ ]]; then
+          last_check_epoch="$value"
+        fi
+        ;;
+      failure_count)
+        if [[ "$value" =~ ^[0-9]+$ ]]; then
+          failure_count="$value"
+        fi
+        ;;
+    esac
+  done <"$state_file"
+fi
+
+write_state() {
+  local next_last_check_epoch="$1"
+  local next_failure_count="$2"
+  local temporary_file="${state_file}.tmp.$$"
+
+  umask 077
+  {
+    printf 'last_check_epoch=%s\n' "$next_last_check_epoch"
+    printf 'failure_count=%s\n' "$next_failure_count"
+  } >"$temporary_file"
+  mv -f "$temporary_file" "$state_file"
+}
+
+execution_window_state="$(
+  EXECUTION_WINDOW_TIMES="$execution_window_times" \
+  EXECUTION_WINDOW_MINUTES="$execution_window_minutes" \
+  EXECUTION_WINDOW_TIMEZONE="$execution_window_timezone" \
+  EXECUTION_WINDOW_NOW_EPOCH="$now_epoch" \
+    python3 - <<'PY'
+from datetime import datetime, timedelta
+from zoneinfo import ZoneInfo
+import os
+
+raw_times = os.environ["EXECUTION_WINDOW_TIMES"].strip()
+window_minutes = int(os.environ["EXECUTION_WINDOW_MINUTES"])
+timezone_name = os.environ["EXECUTION_WINDOW_TIMEZONE"].strip()
+now_epoch = int(os.environ["EXECUTION_WINDOW_NOW_EPOCH"])
+if not raw_times:
+    print("false")
+    raise SystemExit(0)
+
+try:
+    timezone = ZoneInfo(timezone_name)
+except Exception as exc:
+    raise SystemExit(f"invalid execution-window timezone: {timezone_name}") from exc
+
+now = datetime.fromtimestamp(now_epoch, timezone)
+window = timedelta(minutes=window_minutes)
+times = []
+for item in raw_times.split(","):
+    text = item.strip()
+    try:
+        parsed = datetime.strptime(text, "%H:%M").time()
+    except ValueError as exc:
+        raise SystemExit(f"invalid execution-window time: {text}") from exc
+    times.append(parsed)
+
+for day_offset in (-1, 0, 1):
+    date = (now + timedelta(days=day_offset)).date()
+    for scheduled_time in times:
+        scheduled = datetime.combine(date, scheduled_time, timezone)
+        if abs(now - scheduled) <= window:
+            print("true")
+            raise SystemExit(0)
+print("false")
+PY
+)"
+
+case "$execution_window_state" in
+  true)
+    minimum_interval_seconds="$execution_window_interval_seconds"
+    cadence_label="execution-window"
+    ;;
+  false)
+    minimum_interval_seconds="$normal_interval_seconds"
+    cadence_label="normal"
+    ;;
+  *)
+    echo "Unable to determine IB Gateway execution-window cadence." >&2
+    exit 2
+    ;;
+esac
+
+if [ "$last_check_epoch" -gt 0 ] && [ $((now_epoch - last_check_epoch)) -lt "$minimum_interval_seconds" ]; then
+  echo "Skipping IB Gateway readiness probe: ${cadence_label} cadence has not elapsed."
+  exit 0
+fi
+
+echo "Running read-only IB Gateway readiness probe (${cadence_label} cadence)."
+set +e
+IB_GATEWAY_READY_TIMEOUT_SECONDS="$probe_timeout_seconds" \
+IB_GATEWAY_READY_STABILITY_SECONDS=0 \
+  bash "$readiness_script" "$gateway_mode"
+probe_status=$?
+set -e
+
+if [ "$probe_status" -eq 0 ]; then
+  write_state "$now_epoch" 0
+  echo "IB Gateway API readiness probe succeeded."
+  exit 0
+fi
+
+next_failure_count=$((failure_count + 1))
+write_state "$now_epoch" "$next_failure_count"
+if [ "$next_failure_count" -lt "$failure_threshold" ]; then
+  echo "IB Gateway API readiness probe failed (${next_failure_count}/${failure_threshold}); recovery is deferred until the threshold is reached." >&2
+  exit "$probe_status"
+fi
+
+echo "IB Gateway API readiness probe failed ${next_failure_count} consecutive times; starting guarded recovery." >&2
+set +e
+bash "$recovery_script" "$gateway_mode"
+recovery_status=$?
+set -e
+
+if [ "$recovery_status" -eq 0 ]; then
+  write_state "$now_epoch" 0
+  echo "IB Gateway guarded recovery completed successfully."
+else
+  echo "IB Gateway guarded recovery did not complete successfully." >&2
+fi
+exit "$recovery_status"
diff --git a/tests/test_gateway_recovery_scripts.sh b/tests/test_gateway_recovery_scripts.sh
@@ -6,17 +6,20 @@ recover_script="$repo_dir/scripts/recover_ib_gateway_ready.sh"
 swap_script="$repo_dir/scripts/ensure_host_swap.sh"
 daily_restart_script="$repo_dir/scripts/restart_ib_gateway_daily.sh"
 health_watcher_script="$repo_dir/scripts/install_gateway_health_watcher.sh"
+adaptive_monitor_script="$repo_dir/scripts/monitor_ib_gateway_ready.sh"
 unit_helper_script="$repo_dir/scripts/ibkr_gateway_units.sh"
 
 test -f "$recover_script"
 test -f "$swap_script"
 test -f "$daily_restart_script"
 test -f "$health_watcher_script"
+test -f "$adaptive_monitor_script"
 test -f "$unit_helper_script"
 test -x "$recover_script"
 test -x "$swap_script"
 test -x "$daily_restart_script"
 test -x "$health_watcher_script"
+test -x "$adaptive_monitor_script"
 test -x "$unit_helper_script"
 
 grep -Fq 'IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS:-240' "$recover_script"
@@ -64,7 +67,10 @@ grep -Fq '$IBKR_GATEWAY_HEALTHCHECK_SERVICE' "$health_watcher_script"
 grep -Fq '$IBKR_GATEWAY_HEALTHCHECK_TIMER' "$health_watcher_script"
 grep -Fq '$IBKR_GATEWAY_DAILY_RESTART_SERVICE' "$health_watcher_script"
 grep -Fq '$IBKR_GATEWAY_DAILY_RESTART_TIMER' "$health_watcher_script"
-grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300' "$health_watcher_script"
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_TICK_SECONDS:-300' "$health_watcher_script"
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900' "$health_watcher_script"
+grep -Fq 'IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300' "$health_watcher_script"
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2' "$health_watcher_script"
 grep -Fq 'IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC' "$health_watcher_script"
 grep -Fq 'compose_service_name="${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}"' "$health_watcher_script"
 grep -Fq 'gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}"' "$health_watcher_script"
@@ -73,12 +79,52 @@ grep -Fq 'Environment=IB_GATEWAY_COMPOSE_SERVICE_NAME=$compose_service_name' "$h
 grep -Fq 'Environment=COMPOSE_FILE=$compose_file' "$health_watcher_script"
 grep -Fq 'Environment=IB_GATEWAY_RECOVERY_LOCK_FILE=$recovery_lock_file' "$health_watcher_script"
 grep -Fq 'Environment=IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS=0' "$health_watcher_script"
-grep -Fq 'OnActiveSec=$health_interval_seconds' "$health_watcher_script"
+grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=$health_interval_seconds' "$health_watcher_script"
+grep -Fq 'Environment=IB_GATEWAY_EXECUTION_WINDOW_TIMES=$execution_window_times' "$health_watcher_script"
+grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=$failure_threshold' "$health_watcher_script"
+grep -Fq 'monitor_ib_gateway_ready.sh' "$health_watcher_script"
+grep -Fq 'OnActiveSec=$healthcheck_tick_seconds' "$health_watcher_script"
 grep -Fq 'Persistent=false' "$health_watcher_script"
 ! grep -Fq 'OnBootSec=5min' "$health_watcher_script"
 grep -Fq 'enable --now "$IBKR_GATEWAY_HEALTHCHECK_TIMER"' "$health_watcher_script"
 grep -Fq 'enable --now "$IBKR_GATEWAY_DAILY_RESTART_TIMER"' "$health_watcher_script"
 ! grep -Fq 'start ibkr-gateway-healthcheck.service' "$health_watcher_script"
 
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900' "$adaptive_monitor_script"
+grep -Fq 'IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300' "$adaptive_monitor_script"
+grep -Fq 'IB_GATEWAY_EXECUTION_WINDOW_TIMES:-09:45,15:45' "$adaptive_monitor_script"
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2' "$adaptive_monitor_script"
+grep -Fq 'IB_GATEWAY_READY_STABILITY_SECONDS=0' "$adaptive_monitor_script"
+grep -Fq 'recovery is deferred until the threshold is reached' "$adaptive_monitor_script"
+
+tmp_dir="$(mktemp -d)"
+trap 'rm -rf "$tmp_dir"' EXIT
+mkdir -p "$tmp_dir/bin" "$tmp_dir/systemd"
+cat >"$tmp_dir/bin/systemctl" <<'SH'
+#!/usr/bin/env bash
+exit 0
+SH
+chmod +x "$tmp_dir/bin/systemctl"
+
+SYSTEMD_DIR="$tmp_dir/systemd" \
+SYSTEMCTL_BIN="$tmp_dir/bin/systemctl" \
+IB_GATEWAY_HEALTHCHECK_TICK_SECONDS=300 \
+IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=900 \
+IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS=300 \
+IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=2 \
+  bash "$health_watcher_script" live
+
+generated_health_service="$tmp_dir/systemd/ibkr-gateway-healthcheck.service"
+generated_health_timer="$tmp_dir/systemd/ibkr-gateway-healthcheck.timer"
+test -f "$generated_health_service"
+test -f "$generated_health_timer"
+grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=900' "$generated_health_service"
+grep -Fq 'Environment=IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS=300' "$generated_health_service"
+grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=2' "$generated_health_service"
+grep -Fq 'ExecStart=/bin/bash -lc' "$generated_health_service"
+grep -Fq 'monitor_ib_gateway_ready.sh' "$generated_health_service"
+grep -Fq 'OnActiveSec=300' "$generated_health_timer"
+grep -Fq 'OnUnitActiveSec=300' "$generated_health_timer"
+
 grep -Fq 'resolve_ibkr_gateway_unit_suffix()' "$unit_helper_script"
 grep -Fq 'IBKR_2FA_BOT_SERVICE="ibkr-2fa-bot${unit_infix}.service"' "$unit_helper_script"