diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..c813975 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,12 @@ +# Line endings. +# +# The deploy scripts are checked out and run on Linux. With CRLF endings bash +# reads the shebang as `/usr/bin/env bash\r` and refuses the file, so these must +# stay LF regardless of the platform they were committed from. Same for the +# systemd unit bodies embedded in them and for the CI workflow. +*.sh text eol=lf +*.yml text eol=lf +*.yaml text eol=lf + +# Everything else: normalise to LF in the repository, let git decide locally. +* text=auto diff --git a/config.py b/config.py index a49c24c..746b78f 100644 --- a/config.py +++ b/config.py @@ -3,6 +3,9 @@ import os import sys import logging +import platform +import subprocess +from typing import Optional from dotenv import load_dotenv @@ -38,6 +41,59 @@ def configure_logging() -> None: logging.getLogger("discord").setLevel(logging.WARNING) +def _first_line(cmd: list[str], cwd: Optional[str] = None) -> Optional[str]: + """First line of a command's output, or None if it can't be run.""" + try: + result = subprocess.run( + cmd, cwd=cwd, capture_output=True, text=True, timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + if result.returncode != 0: + return None + lines = result.stdout.strip().splitlines() + return lines[0].strip() if lines else None + + +def _ffmpeg_version() -> str: + # "ffmpeg version 6.1.1-3ubuntu5 Copyright (c) 2000-2023 ..." + line = _first_line(["ffmpeg", "-version"]) + parts = line.split() if line else [] + return parts[2] if len(parts) > 2 and parts[1] == "version" else "unknown" + + +def runtime_versions() -> dict: + """ + What this instance is actually running. + + Every lookup degrades to "unknown" rather than failing the boot — a host + deployed without git, or without FFmpeg on PATH, should still start and say + so plainly. + """ + root = os.path.dirname(os.path.abspath(__file__)) + return { + "commit": _first_line(["git", "rev-parse", "--short", "HEAD"], cwd=root) or "unknown", + "yt_dlp": _first_line([sys.executable, "-m", "yt_dlp", "--version"]) or "unknown", + "ffmpeg": _ffmpeg_version(), + "python": platform.python_version(), + } + + +def log_runtime() -> None: + """ + Record the running versions at startup. + + Without this there is no way to tell from `journalctl` which commit or which + yt-dlp was live when something broke — and a stale yt-dlp is the single most + common cause of YouTube failing. + """ + v = runtime_versions() + logging.getLogger("loopify").info( + "Running commit %s — yt-dlp %s, FFmpeg %s, Python %s", + v["commit"], v["yt_dlp"], v["ffmpeg"], v["python"], + ) + + def validate() -> None: """Fail fast with a clear message if required config is missing.""" missing = [name for name, val in { diff --git a/deploy/README.md b/deploy/README.md index 12e0269..95ea7e2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -26,20 +26,22 @@ configured, or follow it step by step. ## 2. Provision it -Copy the project to the instance (excluding secrets/venv) and run the setup -script, which installs `ffmpeg` + Python, builds a venv, and installs the -`loopify-bot` systemd service: +**Clone** the repo on the instance — do not copy the files over. A git checkout +is what makes `deploy/update.sh` work later and lets the bot report which commit +it is running: ```bash -# from your laptop, in the repo root -rsync -av --exclude .venv --exclude .git --exclude __pycache__ \ - -e "ssh -i .pem" ./ ubuntu@:~/LoopifyBot/ - ssh -i .pem ubuntu@ + +git clone https://github.com/Isma-L154/LoopifyBot.git ~/LoopifyBot cd ~/LoopifyBot bash deploy/setup.sh ``` +`setup.sh` installs `ffmpeg` + Python + Deno, builds a venv, and registers two +systemd units: the `loopify-bot` service and a daily `loopify-ytdlp-update` +timer. It is idempotent, so re-running it is safe. + ## 3. Add secrets and start Secrets are **never** committed. Create the `.env` directly on the instance: @@ -56,9 +58,63 @@ sudo journalctl -u loopify-bot -f # live logs — look for "Logged in as .. ```bash ssh -i .pem ubuntu@ -cd ~/LoopifyBot && git pull # or rsync again -sudo systemctl restart loopify-bot +cd ~/LoopifyBot && bash deploy/update.sh +``` + +`update.sh` pulls, reinstalls dependencies **only if `requirements.txt` +changed**, restarts the service, and then verifies it actually came back up — +printing recent logs and failing loudly if it did not. + +### If the host was deployed by copying files instead of cloning + +`update.sh` refuses to run and tells you how to convert it in place. The short +version, from the app directory: + +```bash +git init -b main +git remote add origin https://github.com/Isma-L154/LoopifyBot.git +git fetch origin +git branch --set-upstream-to=origin/main main +git reset --hard origin/main # discards local edits — check first +``` + +The `--set-upstream-to` line matters: without it `update.sh` has nothing to pull +from and stops with an explanation. + +`.env` and `cookies.txt` are gitignored, so they survive this untouched. + +### Knowing what is actually running + +The bot logs its versions at startup, so `journalctl` answers this directly: + ``` +Running commit 0a90877 — yt-dlp 2026.08.19, FFmpeg 6.1.1-3ubuntu5, Python 3.12.3 +``` + +```bash +sudo journalctl -u loopify-bot | grep "Running commit" | tail -1 +``` + +## Keeping yt-dlp current — automatically + +`yt-dlp` is the only dependency deliberately left unpinned. YouTube changes its +player and extractors constantly, so a stale build starts failing to resolve +videos within weeks and fails outright within months — a `2026.3.3` build +returned `HTTP 403` on **every** YouTube URL until it was updated. + +`setup.sh` installs a timer that handles this: + +```bash +systemctl list-timers 'loopify-ytdlp-update*' # when it next runs +sudo systemctl start loopify-ytdlp-update.service # force a refresh now +sudo journalctl -u loopify-ytdlp-update -n 20 # what it did last time +``` + +It runs daily with a randomised delay, restarts the bot **only when the version +actually changed**, and leaves the working version installed if the upgrade +fails — a newer dependency is never worth trading a running bot for. +`Persistent=true` means it catches up after downtime rather than silently +skipping, which matters on a machine that is not on 24/7. ## 🎬 YouTube from cloud IPs — how it's made to work diff --git a/deploy/setup.sh b/deploy/setup.sh index ddc75cf..05d2a8d 100644 --- a/deploy/setup.sh +++ b/deploy/setup.sh @@ -106,11 +106,48 @@ TasksMax=256 WantedBy=multi-user.target UNIT +# ── 5. yt-dlp auto-update timer ─────────────────────────────────────── +# yt-dlp is the one dependency that must NOT be pinned: YouTube changes its +# player constantly and a stale build stops resolving videos within weeks. +echo "==> Installing yt-dlp auto-update timer..." +chmod +x "$APP_DIR/deploy/update-ytdlp.sh" + +sudo tee "/etc/systemd/system/${SERVICE_NAME%-bot}-ytdlp-update.service" >/dev/null </dev/null < Done. Manage the bot with:" echo " sudo systemctl start $SERVICE_NAME" echo " sudo systemctl status $SERVICE_NAME" echo " sudo journalctl -u $SERVICE_NAME -f # live logs" +echo "" +echo " bash deploy/update.sh # pull latest code & restart" +echo " systemctl list-timers '*ytdlp*' # when yt-dlp refreshes next" diff --git a/deploy/update-ytdlp.sh b/deploy/update-ytdlp.sh new file mode 100644 index 0000000..442d3ea --- /dev/null +++ b/deploy/update-ytdlp.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# +# Keep yt-dlp current. Run by loopify-ytdlp-update.timer, not by hand. +# +# YouTube changes its player and extractors constantly, so a yt-dlp that is a +# few weeks old starts failing to resolve videos and a few months old fails +# outright (a 2026.3.3 build returned HTTP 403 on every YouTube URL). Nothing +# else in the venv is upgraded here — the rest is pinned on purpose. +# +# Runs as root so it can restart the service, but drops to the app user for the +# pip install so the venv does not end up owned by root. +# +set -euo pipefail + +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$APP_DIR/.venv" +SERVICE_NAME="loopify-bot" +# The venv's owner is the user the bot runs as. +APP_USER="$(stat -c '%U' "$VENV_DIR")" + +version() { + runuser -u "$APP_USER" -- "$VENV_DIR/bin/python" -m yt_dlp --version 2>/dev/null || echo "none" +} + +before="$(version)" + +# A failed upgrade (offline, PyPI down, a broken release) must leave the working +# version in place. Never trade a running bot for a newer dependency. +if ! runuser -u "$APP_USER" -- "$VENV_DIR/bin/pip" install --quiet --upgrade "yt-dlp[default]"; then + echo "yt-dlp upgrade failed; keeping $before" + exit 0 +fi + +after="$(version)" + +if [[ "$before" == "$after" ]]; then + echo "yt-dlp already current ($after)" + exit 0 +fi + +# Only restart when something actually changed, so playback is never +# interrupted for a no-op. +echo "yt-dlp $before -> $after; restarting $SERVICE_NAME" +systemctl restart "$SERVICE_NAME" diff --git a/deploy/update.sh b/deploy/update.sh new file mode 100644 index 0000000..4bbda80 --- /dev/null +++ b/deploy/update.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# +# Update a deployed bot to the latest committed code and restart it. +# +# Usage, on the host: +# cd ~/LoopifyBot && bash deploy/update.sh +# +set -euo pipefail + +APP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV_DIR="$APP_DIR/.venv" +SERVICE_NAME="loopify-bot" + +cd "$APP_DIR" + +if [[ ! -d .git ]]; then + cat >&2 <<'MSG' +!! This deployment is not a git checkout, so it cannot be updated with git. + + That happens when the code was copied over with rsync. To convert it in + place, from the app directory: + + git init -b main + git remote add origin https://github.com/Isma-L154/LoopifyBot.git + git fetch origin + git branch --set-upstream-to=origin/main main + git reset --hard origin/main # discards local edits — check first! + + Your .env and cookies.txt are gitignored and will not be touched. +MSG + exit 1 +fi + +if ! git rev-parse --abbrev-ref '@{upstream}' >/dev/null 2>&1; then + branch="$(git rev-parse --abbrev-ref HEAD)" + cat >&2 < Fetching..." +git pull --ff-only + +if [[ "$previous" == "$(git rev-parse HEAD)" ]]; then + echo "==> Already up to date." +fi + +# Only rebuild the venv when the pinned set actually changed. +if git diff --quiet "$previous" HEAD -- requirements.txt; then + echo "==> requirements.txt unchanged; skipping dependency install." +else + echo "==> requirements.txt changed; reinstalling dependencies..." + "$VENV_DIR/bin/pip" install --quiet --upgrade -r requirements.txt +fi + +echo "==> Restarting $SERVICE_NAME..." +sudo systemctl restart "$SERVICE_NAME" + +sleep 2 +if sudo systemctl is-active --quiet "$SERVICE_NAME"; then + echo "==> Running commit $(git rev-parse --short HEAD)." +else + echo "!! Service is not active. Recent logs:" >&2 + sudo journalctl -u "$SERVICE_NAME" -n 30 --no-pager + exit 1 +fi diff --git a/main.py b/main.py index 97152df..d4589a1 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ from utils import errors config.configure_logging() +config.log_runtime() config.validate() log = logging.getLogger("loopify") diff --git a/requirements.txt b/requirements.txt index 4fef3c1..d87f9d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,16 @@ -discord.py[voice]>=2.3.0 -yt-dlp[default]>=2024.1.0 -lyricsgenius>=3.0.1 -PyNaCl>=1.5.0 -python-dotenv>=1.0.0 -aiohttp>=3.9.0 -curl_cffi>=0.7.0 \ No newline at end of file +# Pinned so a deploy today installs what was actually tested. Bump these +# deliberately and re-run the suite; do not let them float. +discord.py[voice]==2.7.1 +lyricsgenius==3.12.2 +PyNaCl==1.5.0 +python-dotenv==1.2.2 +aiohttp==3.14.3 +curl_cffi==0.15.0 + +# Deliberately NOT pinned. YouTube changes its player, signature challenge and +# extractors constantly, so a pinned yt-dlp stops resolving videos within weeks +# — a five-month-old build (2026.3.3) returned HTTP 403 on every YouTube URL +# until it was updated. deploy/setup.sh installs a systemd timer that keeps this +# current on the server; the floor below is simply the oldest build known to +# work with the current YouTube player. +yt-dlp[default]>=2026.8.19 diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..41f53e1 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,137 @@ +""" +Runtime version reporting. + +This exists so `journalctl` shows which commit and which yt-dlp were live when +something broke. Its one hard requirement is that it can never prevent the bot +from starting: a host without git, without FFmpeg on PATH, or with a hung +subprocess must still boot and simply report "unknown". +""" + +import subprocess +from unittest.mock import MagicMock + +import pytest + +import config + + +# -- _first_line ------------------------------------------------------- + +def test_first_line_returns_the_first_line_of_output(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=0, stdout="1.2.3\nextra noise\n")) + assert config._first_line(["whatever"]) == "1.2.3" + + +def test_first_line_returns_none_on_a_failing_command(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=128, stdout="")) + assert config._first_line(["git", "rev-parse", "HEAD"]) is None + + +def test_first_line_returns_none_on_empty_output(monkeypatch): + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=0, stdout=" \n")) + assert config._first_line(["whatever"]) is None + + +@pytest.mark.parametrize("boom", [ + FileNotFoundError("no such binary"), + PermissionError("denied"), + subprocess.TimeoutExpired(cmd="git", timeout=5), +]) +def test_first_line_swallows_anything_the_subprocess_throws(monkeypatch, boom): + """A missing binary or a hung command must not take the bot down.""" + def explode(*a, **k): + raise boom + monkeypatch.setattr(subprocess, "run", explode) + assert config._first_line(["git"]) is None + + +def test_first_line_uses_a_timeout(monkeypatch): + """Without one, a hung git call would block startup forever.""" + captured = {} + + def fake_run(cmd, **kwargs): + captured.update(kwargs) + return MagicMock(returncode=0, stdout="ok\n") + + monkeypatch.setattr(subprocess, "run", fake_run) + config._first_line(["git"]) + assert captured.get("timeout") + + +# -- FFmpeg version parsing -------------------------------------------- + +@pytest.mark.parametrize("line,expected", [ + ("ffmpeg version 6.1.1-3ubuntu5 Copyright (c) 2000-2023 the FFmpeg developers", + "6.1.1-3ubuntu5"), + ("ffmpeg version 8.0-essentials_build-www.gyan.dev Copyright (c) 2000-2025", + "8.0-essentials_build-www.gyan.dev"), + ("ffmpeg version n7.0 Copyright", "n7.0"), +]) +def test_ffmpeg_version_is_extracted(monkeypatch, line, expected): + monkeypatch.setattr(config, "_first_line", lambda *a, **k: line) + assert config._ffmpeg_version() == expected + + +@pytest.mark.parametrize("line", [None, "", "something else entirely", "ffmpeg"]) +def test_ffmpeg_version_falls_back_to_unknown(monkeypatch, line): + monkeypatch.setattr(config, "_first_line", lambda *a, **k: line) + assert config._ffmpeg_version() == "unknown" + + +# -- runtime_versions -------------------------------------------------- + +def test_runtime_versions_reports_every_field(): + v = config.runtime_versions() + assert set(v) == {"commit", "yt_dlp", "ffmpeg", "python"} + assert all(isinstance(x, str) and x for x in v.values()) + + +def test_python_version_is_always_real(): + """It comes from the interpreter, so it can never be unknown.""" + assert config.runtime_versions()["python"][0].isdigit() + + +def test_everything_degrades_to_unknown_on_a_bare_host(monkeypatch): + """No git, no FFmpeg, no yt-dlp — the bot must still start.""" + monkeypatch.setattr(config, "_first_line", lambda *a, **k: None) + v = config.runtime_versions() + assert v["commit"] == "unknown" + assert v["yt_dlp"] == "unknown" + assert v["ffmpeg"] == "unknown" + assert v["python"] != "unknown" + + +def test_commit_is_unknown_outside_a_git_checkout(monkeypatch): + """ + The deployed host was rsynced, not cloned, so `git rev-parse` fails there. + That must read as "unknown", not crash the boot. + """ + monkeypatch.setattr(subprocess, "run", lambda *a, **k: MagicMock( + returncode=128, stdout="")) + assert config.runtime_versions()["commit"] == "unknown" + + +# -- log_runtime ------------------------------------------------------- + +def test_log_runtime_writes_one_informative_line(caplog): + with caplog.at_level("INFO", logger="loopify"): + config.log_runtime() + assert "Running commit" in caplog.text + assert "yt-dlp" in caplog.text + + +def test_log_runtime_never_raises(monkeypatch): + """Startup must not depend on version discovery succeeding.""" + monkeypatch.setattr(config, "_first_line", lambda *a, **k: None) + config.log_runtime() + + +def test_log_runtime_leaks_no_secrets(caplog, monkeypatch): + """Nothing from the environment should ever reach the log line.""" + monkeypatch.setattr(config, "DISCORD_TOKEN", "super-secret-token-value") + with caplog.at_level("INFO", logger="loopify"): + config.log_runtime() + assert "super-secret-token-value" not in caplog.text