From 8452a67cb2fe87d01edc91f3308d596118aeeefa Mon Sep 17 00:00:00 2001 From: Ismael Leon Date: Thu, 20 Aug 2026 19:54:26 -0600 Subject: [PATCH 1/2] Make deployment reproducible and self-updating Three related problems with how this bot gets onto a server. Dependencies floated. Every requirement used >=, so two deploys a week apart installed different versions. The live host and the dev box had drifted to lyricsgenius 3.12.2 vs 3.10.0 and aiohttp 3.14.3 vs 3.13.3 from the same file. Everything is now pinned except yt-dlp, which must float: YouTube changes its player constantly and a five-month-old build returned HTTP 403 on every YouTube URL until it was updated. A systemd timer now keeps it current daily, restarting the bot only when the version actually changed and leaving the working version installed if the upgrade fails. The documented update path did not work. deploy/README told you to run git pull, but the host was deployed by rsync and has no .git, so fixes merged to main never reached it - which is why a yt-dlp zombie survived 19 days there after the fix had already landed. Provisioning now clones, update.sh handles updates and refuses to run on a non-git checkout with instructions to convert it, and the bot logs its commit, yt-dlp, FFmpeg and Python versions at startup so journalctl can answer "what is actually running" directly. The shell scripts had CRLF line endings in the repository. That was harmless while deployment was rsync from Windows, but the moment the docs say "git clone" a Linux checkout gets CRLF and bash rejects the shebang. A .gitattributes pins *.sh and *.yml to LF, and the tracked files are renormalised. Verified with shellcheck at style level: clean. Closes #11 Closes #12 --- .gitattributes | 12 ++++ config.py | 56 +++++++++++++++++ deploy/README.md | 70 ++++++++++++++++++--- deploy/setup.sh | 37 +++++++++++ deploy/update-ytdlp.sh | 44 +++++++++++++ deploy/update.sh | 62 +++++++++++++++++++ main.py | 1 + requirements.txt | 23 ++++--- tests/test_config.py | 137 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 426 insertions(+), 16 deletions(-) create mode 100644 .gitattributes create mode 100644 deploy/update-ytdlp.sh create mode 100644 deploy/update.sh create mode 100644 tests/test_config.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c813975 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# Line endings. +# +# The deploy scripts are checked out and run on Linux. With CRLF endings bash +# reads the shebang as `/usr/bin/env bash\r` and refuses the file, so these must +# stay LF regardless of the platform they were committed from. Same for the +# systemd unit bodies embedded in them and for the CI workflow. +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf + +# Everything else: normalise to LF in the repository, let git decide locally. +* text=auto diff --git a/config.py b/config.py index a49c24c..746b78f 100644 --- a/config.py +++ b/config.py @@ -3,6 +3,9 @@ import os import sys import logging +import platform +import subprocess +from typing import Optional from dotenv import load_dotenv @@ -38,6 +41,59 @@ def configure_logging() -> None: logging.getLogger("discord").setLevel(logging.WARNING) +def _first_line(cmd: list[str], cwd: Optional[str] = None) -> Optional[str]: + """First line of a command's output, or None if it can't be run.""" + try: + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + lines = result.stdout.strip().splitlines() + return lines[0].strip() if lines else None + + +def _ffmpeg_version() -> str: + # "ffmpeg version 6.1.1-3ubuntu5 Copyright (c) 2000-2023 ..." + line = _first_line(["ffmpeg", "-version"]) + parts = line.split() if line else [] + return parts[2] if len(parts) > 2 and parts[1] == "version" else "unknown" + + +def runtime_versions() -> dict: + """ + What this instance is actually running. + + Every lookup degrades to "unknown" rather than failing the boot — a host + deployed without git, or without FFmpeg on PATH, should still start and say + so plainly. + """ + root = os.path.dirname(os.path.abspath(__file__)) + return { + "commit": _first_line(["git", "rev-parse", "--short", "HEAD"], cwd=root) or "unknown", + "yt_dlp": _first_line([sys.executable, "-m", "yt_dlp", "--version"]) or "unknown", + "ffmpeg": _ffmpeg_version(), + "python": platform.python_version(), + } + + +def log_runtime() -> None: + """ + Record the running versions at startup. + + Without this there is no way to tell from `journalctl` which commit or which + yt-dlp was live when something broke — and a stale yt-dlp is the single most + common cause of YouTube failing. + """ + v = runtime_versions() + logging.getLogger("loopify").info( + "Running commit %s — yt-dlp %s, FFmpeg %s, Python %s", + v["commit"], v["yt_dlp"], v["ffmpeg"], v["python"], + ) + + def validate() -> None: """Fail fast with a clear message if required config is missing.""" missing = [name for name, val in { diff --git a/deploy/README.md b/deploy/README.md index 12e0269..f127291 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -26,20 +26,22 @@ configured, or follow it step by step. ## 2. Provision it -Copy the project to the instance (excluding secrets/venv) and run the setup -script, which installs `ffmpeg` + Python, builds a venv, and installs the -`loopify-bot` systemd service: +**Clone** the repo on the instance — do not copy the files over. A git checkout +is what makes `deploy/update.sh` work later and lets the bot report which commit +it is running: ```bash -# from your laptop, in the repo root -rsync -av --exclude .venv --exclude .git --exclude __pycache__ \ - -e "ssh -i .pem" ./ ubuntu@:~/LoopifyBot/ - ssh -i .pem ubuntu@ + +git clone https://github.com/Isma-L154/LoopifyBot.git ~/LoopifyBot cd ~/LoopifyBot bash deploy/setup.sh ``` +`setup.sh` installs `ffmpeg` + Python + Deno, builds a venv, and registers two +systemd units: the `loopify-bot` service and a daily `loopify-ytdlp-update` +timer. It is idempotent, so re-running it is safe. + ## 3. Add secrets and start Secrets are **never** committed. Create the `.env` directly on the instance: @@ -56,10 +58,60 @@ sudo journalctl -u loopify-bot -f # live logs — look for "Logged in as .. ```bash ssh -i .pem ubuntu@ -cd ~/LoopifyBot && git pull # or rsync again -sudo systemctl restart loopify-bot +cd ~/LoopifyBot && bash deploy/update.sh ``` +`update.sh` pulls, reinstalls dependencies **only if `requirements.txt` +changed**, restarts the service, and then verifies it actually came back up — +printing recent logs and failing loudly if it did not. + +### If the host was deployed by copying files instead of cloning + +`update.sh` refuses to run and tells you how to convert it in place. The short +version, from the app directory: + +```bash +git init +git remote add origin https://github.com/Isma-L154/LoopifyBot.git +git fetch origin +git reset --hard origin/main # discards local edits — check first +``` + +`.env` and `cookies.txt` are gitignored, so they survive this untouched. + +### Knowing what is actually running + +The bot logs its versions at startup, so `journalctl` answers this directly: + +``` +Running commit 0a90877 — yt-dlp 2026.08.19, FFmpeg 6.1.1-3ubuntu5, Python 3.12.3 +``` + +```bash +sudo journalctl -u loopify-bot | grep "Running commit" | tail -1 +``` + +## Keeping yt-dlp current — automatically + +`yt-dlp` is the only dependency deliberately left unpinned. YouTube changes its +player and extractors constantly, so a stale build starts failing to resolve +videos within weeks and fails outright within months — a `2026.3.3` build +returned `HTTP 403` on **every** YouTube URL until it was updated. + +`setup.sh` installs a timer that handles this: + +```bash +systemctl list-timers 'loopify-ytdlp-update*' # when it next runs +sudo systemctl start loopify-ytdlp-update.service # force a refresh now +sudo journalctl -u loopify-ytdlp-update -n 20 # what it did last time +``` + +It runs daily with a randomised delay, restarts the bot **only when the version +actually changed**, and leaves the working version installed if the upgrade +fails — a newer dependency is never worth trading a running bot for. +`Persistent=true` means it catches up after downtime rather than silently +skipping, which matters on a machine that is not on 24/7. + ## 🎬 YouTube from cloud IPs — how it's made to work YouTube fights bots on **datacenter IPs** (AWS, GCP…) on two fronts, and the bot diff --git a/deploy/setup.sh b/deploy/setup.sh index ddc75cf..05d2a8d 100644 --- a/deploy/setup.sh +++ b/deploy/setup.sh @@ -106,11 +106,48 @@ TasksMax=256 WantedBy=multi-user.target UNIT +# ── 5. yt-dlp auto-update timer ─────────────────────────────────────── +# yt-dlp is the one dependency that must NOT be pinned: YouTube changes its +# player constantly and a stale build stops resolving videos within weeks. +echo "==> Installing yt-dlp auto-update timer..." +chmod +x "$APP_DIR/deploy/update-ytdlp.sh" + +sudo tee "/etc/systemd/system/${SERVICE_NAME%-bot}-ytdlp-update.service" >/dev/null </dev/null < Done. Manage the bot with:" echo " sudo systemctl start $SERVICE_NAME" echo " sudo systemctl status $SERVICE_NAME" echo " sudo journalctl -u $SERVICE_NAME -f # live logs" +echo "" +echo " bash deploy/update.sh # pull latest code & restart" +echo " systemctl list-timers '*ytdlp*' # when yt-dlp refreshes next" diff --git a/deploy/update-ytdlp.sh b/deploy/update-ytdlp.sh new file mode 100644 index 0000000..442d3ea --- /dev/null +++ b/deploy/update-ytdlp.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Keep yt-dlp current. Run by loopify-ytdlp-update.timer, not by hand. +# +# YouTube changes its player and extractors constantly, so a yt-dlp that is a +# few weeks old starts failing to resolve videos and a few months old fails +# outright (a 2026.3.3 build returned HTTP 403 on every YouTube URL). Nothing +# else in the venv is upgraded here — the rest is pinned on purpose. +# +# Runs as root so it can restart the service, but drops to the app user for the +# pip install so the venv does not end up owned by root. +# +set -euo pipefail + +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$APP_DIR/.venv" +SERVICE_NAME="loopify-bot" +# The venv's owner is the user the bot runs as. +APP_USER="$(stat -c '%U' "$VENV_DIR")" + +version() { + runuser -u "$APP_USER" -- "$VENV_DIR/bin/python" -m yt_dlp --version 2>/dev/null || echo "none" +} + +before="$(version)" + +# A failed upgrade (offline, PyPI down, a broken release) must leave the working +# version in place. Never trade a running bot for a newer dependency. +if ! runuser -u "$APP_USER" -- "$VENV_DIR/bin/pip" install --quiet --upgrade "yt-dlp[default]"; then + echo "yt-dlp upgrade failed; keeping $before" + exit 0 +fi + +after="$(version)" + +if [[ "$before" == "$after" ]]; then + echo "yt-dlp already current ($after)" + exit 0 +fi + +# Only restart when something actually changed, so playback is never +# interrupted for a no-op. +echo "yt-dlp $before -> $after; restarting $SERVICE_NAME" +systemctl restart "$SERVICE_NAME" diff --git a/deploy/update.sh b/deploy/update.sh new file mode 100644 index 0000000..9a4e657 --- /dev/null +++ b/deploy/update.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Update a deployed bot to the latest committed code and restart it. +# +# Usage, on the host: +# cd ~/LoopifyBot && bash deploy/update.sh +# +set -euo pipefail + +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$APP_DIR/.venv" +SERVICE_NAME="loopify-bot" + +cd "$APP_DIR" + +if [[ ! -d .git ]]; then + cat >&2 <<'MSG' +!! This deployment is not a git checkout, so it cannot be updated with git. + + That happens when the code was copied over with rsync. To convert it in + place, from the app directory: + + git init + git remote add origin https://github.com/Isma-L154/LoopifyBot.git + git fetch origin + git reset --hard origin/main # discards local edits — check first! + + Your .env and cookies.txt are gitignored and will not be touched. +MSG + exit 1 +fi + +# Record where we were, so the comparison below survives fast-forwards, +# no-op pulls and anything that rewrites the reflog. +previous="$(git rev-parse HEAD)" + +echo "==> Fetching..." +git pull --ff-only + +if [[ "$previous" == "$(git rev-parse HEAD)" ]]; then + echo "==> Already up to date." +fi + +# Only rebuild the venv when the pinned set actually changed. +if git diff --quiet "$previous" HEAD -- requirements.txt; then + echo "==> requirements.txt unchanged; skipping dependency install." +else + echo "==> requirements.txt changed; reinstalling dependencies..." + "$VENV_DIR/bin/pip" install --quiet --upgrade -r requirements.txt +fi + +echo "==> Restarting $SERVICE_NAME..." +sudo systemctl restart "$SERVICE_NAME" + +sleep 2 +if sudo systemctl is-active --quiet "$SERVICE_NAME"; then + echo "==> Running commit $(git rev-parse --short HEAD)." +else + echo "!! Service is not active. Recent logs:" >&2 + sudo journalctl -u "$SERVICE_NAME" -n 30 --no-pager + exit 1 +fi diff --git a/main.py b/main.py index 97152df..d4589a1 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ from utils import errors config.configure_logging() +config.log_runtime() config.validate() log = logging.getLogger("loopify") diff --git a/requirements.txt b/requirements.txt index 4fef3c1..d87f9d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,16 @@ -discord.py[voice]>=2.3.0 -yt-dlp[default]>=2024.1.0 -lyricsgenius>=3.0.1 -PyNaCl>=1.5.0 -python-dotenv>=1.0.0 -aiohttp>=3.9.0 -curl_cffi>=0.7.0 \ No newline at end of file +# Pinned so a deploy today installs what was actually tested. Bump these +# deliberately and re-run the suite; do not let them float. +discord.py[voice]==2.7.1 +lyricsgenius==3.12.2 +PyNaCl==1.5.0 +python-dotenv==1.2.2 +aiohttp==3.14.3 +curl_cffi==0.15.0 + +# Deliberately NOT pinned. YouTube changes its player, signature challenge and +# extractors constantly, so a pinned yt-dlp stops resolving videos within weeks +# — a five-month-old build (2026.3.3) returned HTTP 403 on every YouTube URL +# until it was updated. deploy/setup.sh installs a systemd timer that keeps this +# current on the server; the floor below is simply the oldest build known to +# work with the current YouTube player. +yt-dlp[default]>=2026.8.19 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..41f53e1 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,137 @@ +""" +Runtime version reporting. + +This exists so `journalctl` shows which commit and which yt-dlp were live when +something broke. Its one hard requirement is that it can never prevent the bot +from starting: a host without git, without FFmpeg on PATH, or with a hung +subprocess must still boot and simply report "unknown". +""" + +import subprocess +from unittest.mock import MagicMock + +import pytest + +import config + + +# -- _first_line ------------------------------------------------------- + +def test_first_line_returns_the_first_line_of_output(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=0, stdout="1.2.3\nextra noise\n")) + assert config._first_line(["whatever"]) == "1.2.3" + + +def test_first_line_returns_none_on_a_failing_command(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=128, stdout="")) + assert config._first_line(["git", "rev-parse", "HEAD"]) is None + + +def test_first_line_returns_none_on_empty_output(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=0, stdout=" \n")) + assert config._first_line(["whatever"]) is None + + +@pytest.mark.parametrize("boom", [ + FileNotFoundError("no such binary"), + PermissionError("denied"), + subprocess.TimeoutExpired(cmd="git", timeout=5), +]) +def test_first_line_swallows_anything_the_subprocess_throws(monkeypatch, boom): + """A missing binary or a hung command must not take the bot down.""" + def explode(*a, **k): + raise boom + monkeypatch.setattr(subprocess, "run", explode) + assert config._first_line(["git"]) is None + + +def test_first_line_uses_a_timeout(monkeypatch): + """Without one, a hung git call would block startup forever.""" + captured = {} + + def fake_run(cmd, **kwargs): + captured.update(kwargs) + return MagicMock(returncode=0, stdout="ok\n") + + monkeypatch.setattr(subprocess, "run", fake_run) + config._first_line(["git"]) + assert captured.get("timeout") + + +# -- FFmpeg version parsing -------------------------------------------- + +@pytest.mark.parametrize("line,expected", [ + ("ffmpeg version 6.1.1-3ubuntu5 Copyright (c) 2000-2023 the FFmpeg developers", + "6.1.1-3ubuntu5"), + ("ffmpeg version 8.0-essentials_build-www.gyan.dev Copyright (c) 2000-2025", + "8.0-essentials_build-www.gyan.dev"), + ("ffmpeg version n7.0 Copyright", "n7.0"), +]) +def test_ffmpeg_version_is_extracted(monkeypatch, line, expected): + monkeypatch.setattr(config, "_first_line", lambda *a, **k: line) + assert config._ffmpeg_version() == expected + + +@pytest.mark.parametrize("line", [None, "", "something else entirely", "ffmpeg"]) +def test_ffmpeg_version_falls_back_to_unknown(monkeypatch, line): + monkeypatch.setattr(config, "_first_line", lambda *a, **k: line) + assert config._ffmpeg_version() == "unknown" + + +# -- runtime_versions -------------------------------------------------- + +def test_runtime_versions_reports_every_field(): + v = config.runtime_versions() + assert set(v) == {"commit", "yt_dlp", "ffmpeg", "python"} + assert all(isinstance(x, str) and x for x in v.values()) + + +def test_python_version_is_always_real(): + """It comes from the interpreter, so it can never be unknown.""" + assert config.runtime_versions()["python"][0].isdigit() + + +def test_everything_degrades_to_unknown_on_a_bare_host(monkeypatch): + """No git, no FFmpeg, no yt-dlp — the bot must still start.""" + monkeypatch.setattr(config, "_first_line", lambda *a, **k: None) + v = config.runtime_versions() + assert v["commit"] == "unknown" + assert v["yt_dlp"] == "unknown" + assert v["ffmpeg"] == "unknown" + assert v["python"] != "unknown" + + +def test_commit_is_unknown_outside_a_git_checkout(monkeypatch): + """ + The deployed host was rsynced, not cloned, so `git rev-parse` fails there. + That must read as "unknown", not crash the boot. + """ + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=128, stdout="")) + assert config.runtime_versions()["commit"] == "unknown" + + +# -- log_runtime ------------------------------------------------------- + +def test_log_runtime_writes_one_informative_line(caplog): + with caplog.at_level("INFO", logger="loopify"): + config.log_runtime() + assert "Running commit" in caplog.text + assert "yt-dlp" in caplog.text + + +def test_log_runtime_never_raises(monkeypatch): + """Startup must not depend on version discovery succeeding.""" + monkeypatch.setattr(config, "_first_line", lambda *a, **k: None) + config.log_runtime() + + +def test_log_runtime_leaks_no_secrets(caplog, monkeypatch): + """Nothing from the environment should ever reach the log line.""" + monkeypatch.setattr(config, "DISCORD_TOKEN", "super-secret-token-value") + with caplog.at_level("INFO", logger="loopify"): + config.log_runtime() + assert "super-secret-token-value" not in caplog.text From 61919d6319550254be4abd7cfc3c9ddb72ad38d0 Mon Sep 17 00:00:00 2001 From: Ismael Leon Date: Thu, 20 Aug 2026 19:58:51 -0600 Subject: [PATCH 2/2] Fail clearly in update.sh when no upstream is configured A host converted from a file copy has a git checkout but no tracking branch, so git pull stopped with its own "There is no tracking information" wall of text - in exactly the situation the conversion instructions create. Detect it up front and say what to run. The conversion recipe in both the script and deploy/README now sets the upstream, so following it produces a host that update.sh can actually update. --- deploy/README.md | 6 +++++- deploy/update.sh | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index f127291..95ea7e2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -71,12 +71,16 @@ printing recent logs and failing loudly if it did not. version, from the app directory: ```bash -git init +git init -b main git remote add origin https://github.com/Isma-L154/LoopifyBot.git git fetch origin +git branch --set-upstream-to=origin/main main git reset --hard origin/main # discards local edits — check first ``` +The `--set-upstream-to` line matters: without it `update.sh` has nothing to pull +from and stops with an explanation. + `.env` and `cookies.txt` are gitignored, so they survive this untouched. ### Knowing what is actually running diff --git a/deploy/update.sh b/deploy/update.sh index 9a4e657..4bbda80 100644 --- a/deploy/update.sh +++ b/deploy/update.sh @@ -20,9 +20,10 @@ if [[ ! -d .git ]]; then That happens when the code was copied over with rsync. To convert it in place, from the app directory: - git init + git init -b main git remote add origin https://github.com/Isma-L154/LoopifyBot.git git fetch origin + git branch --set-upstream-to=origin/main main git reset --hard origin/main # discards local edits — check first! Your .env and cookies.txt are gitignored and will not be touched. @@ -30,6 +31,20 @@ MSG exit 1 fi +if ! git rev-parse --abbrev-ref '@{upstream}' >/dev/null 2>&1; then + branch="$(git rev-parse --abbrev-ref HEAD)" + cat >&2 <