From 4876100809df206f90b7ea61495c18760348a8f9 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:26:19 +0800 Subject: [PATCH] feat: add adaptive IB Gateway readiness monitoring Co-Authored-By: Codex --- .github/workflows/ci.yml | 2 + scripts/install_gateway_health_watcher.sh | 33 +++- scripts/monitor_ib_gateway_ready.sh | 184 ++++++++++++++++++++++ tests/test_gateway_recovery_scripts.sh | 50 +++++- tests/test_monitor_ib_gateway_ready.sh | 59 +++++++ 5 files changed, 320 insertions(+), 8 deletions(-) create mode 100755 scripts/monitor_ib_gateway_ready.sh create mode 100755 tests/test_monitor_ib_gateway_ready.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d0f4ed..039297f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,8 @@ jobs: set -euo pipefail bash tests/test_install_2fa_bot_watcher.sh bash tests/test_wait_for_ib_gateway_ready.sh + bash tests/test_gateway_recovery_scripts.sh + bash tests/test_monitor_ib_gateway_ready.sh bash tests/test_workflow_shared_config.sh bash tests/test_docker_compose_ports.sh diff --git a/scripts/install_gateway_health_watcher.sh b/scripts/install_gateway_health_watcher.sh index 2470c13..064a882 100755 --- a/scripts/install_gateway_health_watcher.sh +++ b/scripts/install_gateway_health_watcher.sh @@ -10,7 +10,14 @@ compose_service_name="${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}" unit_suffix="${IB_GATEWAY_UNIT_SUFFIX:-}" gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}" compose_file="${COMPOSE_FILE:-docker-compose.yml}" -health_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300}" +healthcheck_tick_seconds="${IB_GATEWAY_HEALTHCHECK_TICK_SECONDS:-300}" +health_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900}" +execution_window_interval_seconds="${IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300}" +execution_window_times="${IB_GATEWAY_EXECUTION_WINDOW_TIMES:-09:45,15:45}" +execution_window_minutes="${IB_GATEWAY_EXECUTION_WINDOW_MINUTES:-60}" +execution_window_timezone="${IB_GATEWAY_EXECUTION_WINDOW_TIMEZONE:-America/New_York}" +failure_threshold="${IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2}" +probe_timeout_seconds="${IB_GATEWAY_HEALTHCHECK_PROBE_TIMEOUT_SECONDS:-30}" daily_restart_calendar="${IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC}" . "$script_dir/ibkr_gateway_units.sh" @@ -22,12 +29,18 @@ else default_lock_file="/var/lock/ib_gateway_recovery.lock" fi recovery_lock_file="${IB_GATEWAY_RECOVERY_LOCK_FILE:-$default_lock_file}" +if [ -n "$resolved_unit_suffix" ]; then + default_state_file="/var/lib/ib_gateway_healthcheck/${resolved_unit_suffix}.state" +else + default_state_file="/var/lib/ib_gateway_healthcheck/default.state" +fi +state_file="${IB_GATEWAY_HEALTHCHECK_STATE_FILE:-$default_state_file}" install -d "$systemd_dir" cat >"$systemd_dir/$IBKR_GATEWAY_HEALTHCHECK_SERVICE" <"$systemd_dir/$IBKR_GATEWAY_HEALTHCHECK_TIMER" <&2 + exit 2 + fi +done + +if [ -n "$configured_now_epoch" ]; then + if ! [[ "$configured_now_epoch" =~ ^[0-9]+$ ]]; then + echo "IB_GATEWAY_HEALTHCHECK_NOW_EPOCH must be a non-negative integer." >&2 + exit 2 + fi + now_epoch="$configured_now_epoch" +else + now_epoch="$(date +%s)" +fi + +state_dir="$(dirname "$state_file")" +install -d -m 700 "$state_dir" + +last_check_epoch=0 +failure_count=0 +if [ -f "$state_file" ]; then + while IFS='=' read -r key value; do + case "$key" in + last_check_epoch) + if [[ "$value" =~ ^[0-9]+$ ]]; then + last_check_epoch="$value" + fi + ;; + failure_count) + if [[ "$value" =~ ^[0-9]+$ ]]; then + failure_count="$value" + fi + ;; + esac + done <"$state_file" +fi + +write_state() { + local next_last_check_epoch="$1" + local next_failure_count="$2" + local temporary_file="${state_file}.tmp.$$" + + umask 077 + { + printf 'last_check_epoch=%s\n' "$next_last_check_epoch" + printf 'failure_count=%s\n' "$next_failure_count" + } >"$temporary_file" + mv -f "$temporary_file" "$state_file" +} + +execution_window_state="$( + EXECUTION_WINDOW_TIMES="$execution_window_times" \ + EXECUTION_WINDOW_MINUTES="$execution_window_minutes" \ + EXECUTION_WINDOW_TIMEZONE="$execution_window_timezone" \ + EXECUTION_WINDOW_NOW_EPOCH="$now_epoch" \ + python3 - <<'PY' +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo +import os + +raw_times = os.environ["EXECUTION_WINDOW_TIMES"].strip() +window_minutes = int(os.environ["EXECUTION_WINDOW_MINUTES"]) +timezone_name = os.environ["EXECUTION_WINDOW_TIMEZONE"].strip() +now_epoch = int(os.environ["EXECUTION_WINDOW_NOW_EPOCH"]) +if not raw_times: + print("false") + raise SystemExit(0) + +try: + timezone = ZoneInfo(timezone_name) +except Exception as exc: + raise SystemExit(f"invalid execution-window timezone: {timezone_name}") from exc + +now = datetime.fromtimestamp(now_epoch, timezone) +window = timedelta(minutes=window_minutes) +times = [] +for item in raw_times.split(","): + text = item.strip() + try: + parsed = datetime.strptime(text, "%H:%M").time() + except ValueError as exc: + raise SystemExit(f"invalid execution-window time: {text}") from exc + times.append(parsed) + +for day_offset in (-1, 0, 1): + date = (now + timedelta(days=day_offset)).date() + for scheduled_time in times: + scheduled = datetime.combine(date, scheduled_time, timezone) + if abs(now - scheduled) <= window: + print("true") + raise SystemExit(0) +print("false") +PY +)" + +case "$execution_window_state" in + true) + minimum_interval_seconds="$execution_window_interval_seconds" + cadence_label="execution-window" + ;; + false) + minimum_interval_seconds="$normal_interval_seconds" + cadence_label="normal" + ;; + *) + echo "Unable to determine IB Gateway execution-window cadence." >&2 + exit 2 + ;; +esac + +if [ "$last_check_epoch" -gt 0 ] && [ $((now_epoch - last_check_epoch)) -lt "$minimum_interval_seconds" ]; then + echo "Skipping IB Gateway readiness probe: ${cadence_label} cadence has not elapsed." + exit 0 +fi + +echo "Running read-only IB Gateway readiness probe (${cadence_label} cadence)." +set +e +IB_GATEWAY_READY_TIMEOUT_SECONDS="$probe_timeout_seconds" \ +IB_GATEWAY_READY_STABILITY_SECONDS=0 \ + bash "$readiness_script" "$gateway_mode" +probe_status=$? +set -e + +if [ "$probe_status" -eq 0 ]; then + write_state "$now_epoch" 0 + echo "IB Gateway API readiness probe succeeded." + exit 0 +fi + +next_failure_count=$((failure_count + 1)) +write_state "$now_epoch" "$next_failure_count" +if [ "$next_failure_count" -lt "$failure_threshold" ]; then + echo "IB Gateway API readiness probe failed (${next_failure_count}/${failure_threshold}); recovery is deferred until the threshold is reached." >&2 + exit "$probe_status" +fi + +echo "IB Gateway API readiness probe failed ${next_failure_count} consecutive times; starting guarded recovery." >&2 +set +e +bash "$recovery_script" "$gateway_mode" +recovery_status=$? +set -e + +if [ "$recovery_status" -eq 0 ]; then + write_state "$now_epoch" 0 + echo "IB Gateway guarded recovery completed successfully." +else + echo "IB Gateway guarded recovery did not complete successfully." >&2 +fi +exit "$recovery_status" diff --git a/tests/test_gateway_recovery_scripts.sh b/tests/test_gateway_recovery_scripts.sh index 09f7b27..74d2294 100644 --- a/tests/test_gateway_recovery_scripts.sh +++ b/tests/test_gateway_recovery_scripts.sh @@ -6,17 +6,20 @@ recover_script="$repo_dir/scripts/recover_ib_gateway_ready.sh" swap_script="$repo_dir/scripts/ensure_host_swap.sh" daily_restart_script="$repo_dir/scripts/restart_ib_gateway_daily.sh" health_watcher_script="$repo_dir/scripts/install_gateway_health_watcher.sh" +adaptive_monitor_script="$repo_dir/scripts/monitor_ib_gateway_ready.sh" unit_helper_script="$repo_dir/scripts/ibkr_gateway_units.sh" test -f "$recover_script" test -f "$swap_script" test -f "$daily_restart_script" test -f "$health_watcher_script" +test -f "$adaptive_monitor_script" test -f "$unit_helper_script" test -x "$recover_script" test -x "$swap_script" test -x "$daily_restart_script" test -x "$health_watcher_script" +test -x "$adaptive_monitor_script" test -x "$unit_helper_script" grep -Fq 'IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS:-240' "$recover_script" @@ -64,7 +67,10 @@ grep -Fq '$IBKR_GATEWAY_HEALTHCHECK_SERVICE' "$health_watcher_script" grep -Fq '$IBKR_GATEWAY_HEALTHCHECK_TIMER' "$health_watcher_script" grep -Fq '$IBKR_GATEWAY_DAILY_RESTART_SERVICE' "$health_watcher_script" grep -Fq '$IBKR_GATEWAY_DAILY_RESTART_TIMER' "$health_watcher_script" -grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_TICK_SECONDS:-300' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2' "$health_watcher_script" grep -Fq 'IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC' "$health_watcher_script" grep -Fq 'compose_service_name="${IB_GATEWAY_COMPOSE_SERVICE_NAME:-ib-gateway}"' "$health_watcher_script" grep -Fq 'gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}"' "$health_watcher_script" @@ -73,12 +79,52 @@ grep -Fq 'Environment=IB_GATEWAY_COMPOSE_SERVICE_NAME=$compose_service_name' "$h grep -Fq 'Environment=COMPOSE_FILE=$compose_file' "$health_watcher_script" grep -Fq 'Environment=IB_GATEWAY_RECOVERY_LOCK_FILE=$recovery_lock_file' "$health_watcher_script" grep -Fq 'Environment=IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS=0' "$health_watcher_script" -grep -Fq 'OnActiveSec=$health_interval_seconds' "$health_watcher_script" +grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=$health_interval_seconds' "$health_watcher_script" +grep -Fq 'Environment=IB_GATEWAY_EXECUTION_WINDOW_TIMES=$execution_window_times' "$health_watcher_script" +grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=$failure_threshold' "$health_watcher_script" +grep -Fq 'monitor_ib_gateway_ready.sh' "$health_watcher_script" +grep -Fq 'OnActiveSec=$healthcheck_tick_seconds' "$health_watcher_script" grep -Fq 'Persistent=false' "$health_watcher_script" ! grep -Fq 'OnBootSec=5min' "$health_watcher_script" grep -Fq 'enable --now "$IBKR_GATEWAY_HEALTHCHECK_TIMER"' "$health_watcher_script" grep -Fq 'enable --now "$IBKR_GATEWAY_DAILY_RESTART_TIMER"' "$health_watcher_script" ! grep -Fq 'start ibkr-gateway-healthcheck.service' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-900' "$adaptive_monitor_script" +grep -Fq 'IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS:-300' "$adaptive_monitor_script" +grep -Fq 'IB_GATEWAY_EXECUTION_WINDOW_TIMES:-09:45,15:45' "$adaptive_monitor_script" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD:-2' "$adaptive_monitor_script" +grep -Fq 'IB_GATEWAY_READY_STABILITY_SECONDS=0' "$adaptive_monitor_script" +grep -Fq 'recovery is deferred until the threshold is reached' "$adaptive_monitor_script" + +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT +mkdir -p "$tmp_dir/bin" "$tmp_dir/systemd" +cat >"$tmp_dir/bin/systemctl" <<'SH' +#!/usr/bin/env bash +exit 0 +SH +chmod +x "$tmp_dir/bin/systemctl" + +SYSTEMD_DIR="$tmp_dir/systemd" \ +SYSTEMCTL_BIN="$tmp_dir/bin/systemctl" \ +IB_GATEWAY_HEALTHCHECK_TICK_SECONDS=300 \ +IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=900 \ +IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS=300 \ +IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=2 \ + bash "$health_watcher_script" live + +generated_health_service="$tmp_dir/systemd/ibkr-gateway-healthcheck.service" +generated_health_timer="$tmp_dir/systemd/ibkr-gateway-healthcheck.timer" +test -f "$generated_health_service" +test -f "$generated_health_timer" +grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=900' "$generated_health_service" +grep -Fq 'Environment=IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS=300' "$generated_health_service" +grep -Fq 'Environment=IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=2' "$generated_health_service" +grep -Fq 'ExecStart=/bin/bash -lc' "$generated_health_service" +grep -Fq 'monitor_ib_gateway_ready.sh' "$generated_health_service" +grep -Fq 'OnActiveSec=300' "$generated_health_timer" +grep -Fq 'OnUnitActiveSec=300' "$generated_health_timer" + grep -Fq 'resolve_ibkr_gateway_unit_suffix()' "$unit_helper_script" grep -Fq 'IBKR_2FA_BOT_SERVICE="ibkr-2fa-bot${unit_infix}.service"' "$unit_helper_script" diff --git a/tests/test_monitor_ib_gateway_ready.sh b/tests/test_monitor_ib_gateway_ready.sh new file mode 100755 index 0000000..1069d00 --- /dev/null +++ b/tests/test_monitor_ib_gateway_ready.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_dir="$(cd "$(dirname "$0")/.." && pwd)" +monitor_script="$repo_dir/scripts/monitor_ib_gateway_ready.sh" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "$tmp_dir"' EXIT + +readiness_script="$tmp_dir/readiness.sh" +recovery_script="$tmp_dir/recovery.sh" +state_file="$tmp_dir/state" +call_log="$tmp_dir/calls.log" + +cat >"$readiness_script" <<'SH' +#!/usr/bin/env bash +printf 'probe\n' >>"$CALL_LOG" +exit "${PROBE_EXIT_CODE:-0}" +SH +cat >"$recovery_script" <<'SH' +#!/usr/bin/env bash +printf 'recover\n' >>"$CALL_LOG" +exit "${RECOVERY_EXIT_CODE:-0}" +SH +chmod +x "$readiness_script" "$recovery_script" + +base_env=( + CALL_LOG="$call_log" + IB_GATEWAY_READINESS_SCRIPT="$readiness_script" + IB_GATEWAY_RECOVERY_SCRIPT="$recovery_script" + IB_GATEWAY_HEALTHCHECK_STATE_FILE="$state_file" + IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS=900 + IB_GATEWAY_EXECUTION_WINDOW_INTERVAL_SECONDS=300 + IB_GATEWAY_EXECUTION_WINDOW_TIMES=09:45 + IB_GATEWAY_EXECUTION_WINDOW_MINUTES=60 + IB_GATEWAY_EXECUTION_WINDOW_TIMEZONE=America/New_York + IB_GATEWAY_HEALTHCHECK_FAILURE_THRESHOLD=2 +) + +env "${base_env[@]}" IB_GATEWAY_HEALTHCHECK_NOW_EPOCH=1787832000 \ + bash "$monitor_script" live +test "$(tr '\n' ' ' <"$call_log")" = "probe " +grep -Fxq 'failure_count=0' "$state_file" + +env "${base_env[@]}" IB_GATEWAY_HEALTHCHECK_NOW_EPOCH=1787832300 \ + bash "$monitor_script" live +test "$(wc -l <"$call_log")" -eq 1 + +if env "${base_env[@]}" PROBE_EXIT_CODE=1 IB_GATEWAY_HEALTHCHECK_NOW_EPOCH=1787838300 \ + bash "$monitor_script" live; then + echo "First failed readiness probe unexpectedly succeeded." >&2 + exit 1 +fi +test "$(tr '\n' ' ' <"$call_log")" = "probe probe " +grep -Fxq 'failure_count=1' "$state_file" + +env "${base_env[@]}" PROBE_EXIT_CODE=1 IB_GATEWAY_HEALTHCHECK_NOW_EPOCH=1787838600 \ + bash "$monitor_script" live +test "$(tr '\n' ' ' <"$call_log")" = "probe probe probe recover " +grep -Fxq 'failure_count=0' "$state_file"