Keep yt-dlp current at runtime, with the bundled copy as fallback #12

Merged
branderson merged 1 commits from ytdlp-runtime into main 2026-10-09 16:43:25 -07:00
7 changed files with 457 additions and 115 deletions
+14 -11
View File
@@ -74,7 +74,7 @@ See `config.example.toml` for every available field (`whisper_url`,
`whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`, `whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`,
`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`, `creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`,
`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`, `ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`,
`ytdlp_pot_provider_url`). `ytdlp_pot_provider_url`, `ytdlp_version`).
`whisper_api` selects the wire format: `"openai"` (default) speaks the `whisper_api` selects the wire format: `"openai"` (default) speaks the
OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"` OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"`
@@ -182,21 +182,24 @@ still wins.
`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the `HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
page but refused the media URL. In rough order of likelihood: page but refused the media URL. In rough order of likelihood:
1. **yt-dlp is stale.** YouTube changes its defences every few weeks and the 1. **yt-dlp is stale.** YouTube changes its defences every few weeks, so
version baked into the image does not. Hotfix a running container, then songtrace keeps yt-dlp current itself. Once a day, and again whenever a
rebuild so it sticks: download fails outright, it checks PyPI for a newer stable release,
installs it under `$XDG_CACHE_HOME/songtrace/` (`/data/models/cache` in the
container), and tries it before the copy installed with songtrace. That
copy is never modified and is always the fallback, so a release that breaks
compatibility costs a retry rather than the download. The job log names
the yt-dlp version behind each attempt.
```sh `ytdlp_version` overrides this: `"bundled"` uses only the installed copy,
docker compose exec songtrace pip install -U 'yt-dlp[default,curl-cffi,deno]' and a release number such as `"2026.08.19"` pins that release instead of
docker compose restart songtrace the newest.
docker compose build --pull songtrace # permanent
```
2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine 2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine
to solve YouTube's player challenges; without them it silently falls back to to solve YouTube's player challenges; without them it silently falls back to
formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml` formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml`
cover this. Check with cover this, for both the installed copy and the ones fetched at runtime.
`docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug Check with `docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
header must show a `JS runtimes:` line that is not `none`. header must show a `JS runtimes:` line that is not `none`.
3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get 3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get
+3
View File
@@ -18,3 +18,6 @@ bind_host = "0.0.0.0"
# ytdlp_cookies = "/config/youtube-cookies.txt" # Netscape cookie jar # ytdlp_cookies = "/config/youtube-cookies.txt" # Netscape cookie jar
# ytdlp_impersonate = "chrome" # TLS fingerprint to present # ytdlp_impersonate = "chrome" # TLS fingerprint to present
# ytdlp_pot_provider_url = "http://bgutil-provider:4416" # ytdlp_pot_provider_url = "http://bgutil-provider:4416"
# Newer yt-dlp releases are fetched from PyPI at runtime and tried before the
# bundled one; "bundled" turns that off, a release number pins one.
# ytdlp_version = "auto"
+3
View File
@@ -24,6 +24,9 @@ class Config:
ytdlp_cookies: Path | None = None ytdlp_cookies: Path | None = None
ytdlp_impersonate: str | None = None ytdlp_impersonate: str | None = None
ytdlp_pot_provider_url: str | None = None ytdlp_pot_provider_url: str | None = None
# "auto" keeps yt-dlp current from PyPI, "bundled" uses only the copy
# installed with songtrace, and a release such as "2026.08.19" pins that.
ytdlp_version: str = "auto"
_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"} _PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"}
+34 -50
View File
@@ -2,6 +2,7 @@ import json
import shutil import shutil
from pathlib import Path from pathlib import Path
from .. import ytdlp
from ..config import Config from ..config import Config
from ..media import ffmpeg from ..media import ffmpeg
from ..pipeline import Ctx, PipelineError, Stage from ..pipeline import Ctx, PipelineError, Stage
@@ -22,67 +23,50 @@ def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
return video, thumbs[0] if thumbs else None return video, thumbs[0] if thumbs else None
class _YdlLog: def _ydl_args(cfg: Config, dest: Path, client: str, url: str) -> list[str]:
"""Routes yt-dlp's own messages into the job log, minus progress noise.""" args = [
"--output", str(dest / "source_video.%(ext)s"),
def __init__(self, log): "--format", "bestvideo*+bestaudio/best",
self._log = log "--write-thumbnail", "--no-playlist", "--no-progress",
"--retries", "10", "--fragment-retries", "10",
def debug(self, msg: str) -> None: "--extractor-retries", "5",
if not msg.startswith("[debug] "): ]
self._log(msg)
def warning(self, msg: str) -> None:
self._log(msg)
error = warning
def _ydl_opts(cfg: Config, dest: Path, client: str, log) -> dict:
extractor_args: dict[str, dict[str, list[str]]] = {}
if client != "default": if client != "default":
extractor_args["youtube"] = {"player_client": [client]} args += ["--extractor-args", f"youtube:player_client={client}"]
if cfg.ytdlp_pot_provider_url: if cfg.ytdlp_pot_provider_url:
extractor_args["youtubepot-bgutilhttp"] = { args += ["--extractor-args",
"base_url": [cfg.ytdlp_pot_provider_url]} f"youtubepot-bgutilhttp:base_url={cfg.ytdlp_pot_provider_url}"]
opts = {
"outtmpl": str(dest / "source_video.%(ext)s"),
"format": "bestvideo*+bestaudio/best",
"writethumbnail": True,
"noplaylist": True,
"noprogress": True,
"logger": _YdlLog(log),
"retries": 10,
"fragment_retries": 10,
"extractor_retries": 5,
"extractor_args": extractor_args,
}
if cfg.ytdlp_cookies: if cfg.ytdlp_cookies:
opts["cookiefile"] = str(cfg.ytdlp_cookies) args += ["--cookies", str(cfg.ytdlp_cookies)]
if cfg.ytdlp_impersonate: if cfg.ytdlp_impersonate:
from yt_dlp.networking.impersonate import ImpersonateTarget args += ["--impersonate", cfg.ytdlp_impersonate]
opts["impersonate"] = ImpersonateTarget.from_str(cfg.ytdlp_impersonate) return [*args, "--", url]
return opts
def download_youtube(url: str, dest: Path, cfg: Config, def download_youtube(url: str, dest: Path, cfg: Config,
log=print) -> tuple[Path, Path | None]: log=print) -> tuple[Path, Path | None]:
import yt_dlp
clients = list(cfg.ytdlp_clients) or ["default"] clients = list(cfg.ytdlp_clients) or ["default"]
tried: list[str] = []
last: Exception | None = None last: Exception | None = None
for client in clients: # If every copy fails, look for a release newer than the daily check found.
try: for refresh in (False, True):
with yt_dlp.YoutubeDL(_ydl_opts(cfg, dest, client, log)) as ydl: for ydl in ytdlp.candidates(cfg, log, refresh):
ydl.extract_info(url, download=True) if ydl.version in tried:
return _classify_downloads(dest) continue
except Exception as e: tried.append(ydl.version)
last = e for client in clients:
log(f"yt-dlp player_client={client} failed: {e}") try:
ydl.run(_ydl_args(cfg, dest, client, url), log)
return _classify_downloads(dest)
except PipelineError as e:
last = e
log(f"yt-dlp {ydl.version} player_client={client} "
f"failed: {e}")
raise PipelineError( raise PipelineError(
f"yt-dlp {yt_dlp.version.__version__} could not download {url} after " f"yt-dlp {', '.join(tried)} could not download {url} after trying "
f"trying player_client={', '.join(clients)}. Last error: {last}. " f"player_client={', '.join(clients)}. Last error: {last}. "
"YouTube blocks change often: upgrade yt-dlp, then see the yt-dlp " "See the yt-dlp options in config.toml (cookies, impersonation, "
"options in config.toml (cookies, impersonation, PO token provider).") "PO token provider).")
def _run(ctx: Ctx) -> None: def _run(ctx: Ctx) -> None:
+185
View File
@@ -0,0 +1,185 @@
"""The one place songtrace touches yt-dlp.
YouTube breaks old yt-dlp releases faster than songtrace is released, so
besides the copy installed with songtrace (the bundled one), newer stable
releases are fetched from PyPI into a cache dir at runtime and tried first,
with the bundled copy as the fallback. yt-dlp runs in a subprocess and is
driven through its command line, its most stable interface, so a cached
release never mixes with anything songtrace has imported.
"""
import importlib.metadata
import os
import re
import shutil
import subprocess
import sys
import tempfile
import threading
import time
from dataclasses import dataclass
from pathlib import Path
import httpx
from .config import Config
from .pipeline import PipelineError
# The extras are what make YouTube work; see pyproject.toml.
REQUIREMENT = "yt-dlp[default,curl-cffi,deno]"
CHECK_INTERVAL = 24 * 3600
_lock = threading.Lock()
class YtDlpError(PipelineError):
pass
def _key(version: str) -> tuple[int, ...]:
return tuple(int(p) for p in version.split("."))
def cache_dir() -> Path:
base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
# Installs hold compiled wheels, so they are only good for one Python.
return Path(base) / "songtrace" / f"yt-dlp-py{sys.version_info[0]}{sys.version_info[1]}"
@dataclass(frozen=True)
class YtDlp:
version: str
path: Path | None = None # None for the bundled copy
def _env(self) -> dict[str, str]:
env = dict(os.environ)
if self.path:
env["PYTHONPATH"] = os.pathsep.join(
filter(None, [str(self.path), env.get("PYTHONPATH")]))
env["PATH"] = os.pathsep.join(
[str(self.path / "bin"), env.get("PATH", "")])
return env
def run(self, args: list[str], log) -> None:
"""Runs yt-dlp with `args`, passing each line it prints to `log`."""
proc = subprocess.Popen(
[sys.executable, "-P", "-m", "yt_dlp", "--ignore-config", *args],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
errors="replace", env=self._env())
last = error = ""
for line in proc.stdout:
last = line.rstrip()
log(last)
if last.startswith("ERROR:"):
error = last
if proc.wait() == 0:
return
if proc.returncode == 2:
raise YtDlpError(f"yt-dlp {self.version} rejected songtrace's "
f"options, so it may be incompatible: {last}")
raise YtDlpError(error or f"yt-dlp exited with status {proc.returncode}")
def bundled() -> YtDlp:
return YtDlp(importlib.metadata.version("yt-dlp"))
def _latest_release() -> str:
r = httpx.get("https://pypi.org/pypi/yt-dlp/json", timeout=15,
follow_redirects=True)
r.raise_for_status()
return r.json()["info"]["version"]
def _pip_install(target: Path, version: str) -> None:
proc = subprocess.run(
[sys.executable, "-m", "pip", "install", "--quiet", "--no-input",
"--disable-pip-version-check", "--target", str(target),
f"{REQUIREMENT}=={version}"],
capture_output=True, text=True)
if proc.returncode:
raise YtDlpError(f"pip could not install yt-dlp {version}: "
f"{proc.stderr.strip()[-500:]}")
def _install(version: str, log) -> YtDlp:
"""Installs `version` into the cache unless it is already there.
The install is checked to run before it is moved into place, so a
directory in the cache is always a usable yt-dlp."""
dest = cache_dir() / version
if dest.is_dir():
return YtDlp(version, dest)
dest.parent.mkdir(parents=True, exist_ok=True)
log(f"installing yt-dlp {version}")
tmp = Path(tempfile.mkdtemp(dir=dest.parent, prefix=".tmp-"))
try:
_pip_install(tmp, version)
proc = subprocess.run(
[sys.executable, "-P", "-m", "yt_dlp", "--version"],
capture_output=True, text=True, env=YtDlp(version, tmp)._env())
got = proc.stdout.strip()
if proc.returncode or _key(got) != _key(version):
raise YtDlpError(f"yt-dlp {version} failed its check after install: "
f"{(got or proc.stderr.strip())[-500:]}")
try:
tmp.rename(dest)
except OSError:
if not dest.is_dir():
raise
finally:
shutil.rmtree(tmp, ignore_errors=True)
return YtDlp(version, dest)
def _cached() -> list[YtDlp]:
root = cache_dir()
if not root.is_dir():
return []
found = [YtDlp(p.name, p) for p in root.iterdir()
if p.is_dir() and re.fullmatch(r"\d+(\.\d+)*", p.name)]
return sorted(found, key=lambda y: _key(y.version))
def _newest(base: YtDlp, refresh: bool, log) -> YtDlp | None:
"""The newest cached release newer than `base`, checking PyPI for a newer
one first if that has not been done for a day or `refresh` is set."""
marker = cache_dir() / ".checked"
if (refresh or not marker.exists()
or time.time() - marker.stat().st_mtime > CHECK_INTERVAL):
# Recorded even on failure, so a broken release is not reinstalled
# for every job; a failed download refreshes regardless.
marker.parent.mkdir(parents=True, exist_ok=True)
marker.touch()
try:
latest = _latest_release()
if _key(latest) > _key(base.version):
_install(latest, log)
except Exception as e:
log(f"could not update yt-dlp: {e}")
newer = [y for y in _cached() if _key(y.version) > _key(base.version)]
for old in newer[:-1]:
shutil.rmtree(old.path, ignore_errors=True)
return newer[-1] if newer else None
def candidates(cfg: Config, log, refresh: bool = False) -> list[YtDlp]:
"""The yt-dlp copies to try, in order: the newest release (or the one
`cfg.ytdlp_version` pins), then the bundled copy."""
base = bundled()
want = cfg.ytdlp_version
if want == "bundled":
return [base]
with _lock:
if want == "auto":
first = _newest(base, refresh, log)
elif not re.fullmatch(r"\d+(\.\d+)*", want):
raise YtDlpError(f"ytdlp_version must be \"auto\", \"bundled\" or a "
f"release such as \"2026.08.19\", not {want!r}")
elif _key(want) == _key(base.version):
first = None
else:
try:
first = _install(want, log)
except Exception as e:
log(f"could not install the pinned yt-dlp {want}: {e}")
first = None
return [first, base] if first else [base]
+79 -54
View File
@@ -91,87 +91,112 @@ def test_classify_ignores_partials_left_by_a_failed_attempt(tmp_path):
video, _ = acquire._classify_downloads(tmp_path) video, _ = acquire._classify_downloads(tmp_path)
assert video.name == "source_video.webm" assert video.name == "source_video.webm"
def test_ydl_opts_default_client_adds_no_player_client(tmp_path): def test_ydl_args_default_client_adds_no_player_client(tmp_path):
cfg = Config(whisper_url="http://x/v1") cfg = Config(whisper_url="http://x/v1")
opts = acquire._ydl_opts(cfg, tmp_path, "default", print) args = acquire._ydl_args(cfg, tmp_path, "default", "https://yt/x")
assert "youtube" not in opts["extractor_args"] assert "--extractor-args" not in args
assert opts["retries"] and opts["fragment_retries"] and opts["extractor_retries"] assert "--retries" in args and "--fragment-retries" in args
assert args[-2:] == ["--", "https://yt/x"]
def test_ydl_opts_named_client_and_pot_provider(tmp_path): def test_ydl_args_named_client_pot_provider_and_cookies(tmp_path):
cfg = Config(whisper_url="http://x/v1", cfg = Config(whisper_url="http://x/v1",
ytdlp_pot_provider_url="http://pot:4416", ytdlp_pot_provider_url="http://pot:4416",
ytdlp_cookies=tmp_path / "cookies.txt") ytdlp_cookies=tmp_path / "cookies.txt",
opts = acquire._ydl_opts(cfg, tmp_path, "tv", print) ytdlp_impersonate="chrome")
assert opts["extractor_args"]["youtube"] == {"player_client": ["tv"]} args = acquire._ydl_args(cfg, tmp_path, "tv", "https://yt/x")
assert opts["extractor_args"]["youtubepot-bgutilhttp"] == { joined = " ".join(args)
"base_url": ["http://pot:4416"]} assert "--extractor-args youtube:player_client=tv" in joined
assert opts["cookiefile"] == str(tmp_path / "cookies.txt") assert ("--extractor-args youtubepot-bgutilhttp:base_url=http://pot:4416"
in joined)
assert f"--cookies {tmp_path / 'cookies.txt'}" in joined
assert "--impersonate chrome" in joined
def _fake_yt_dlp(monkeypatch, behaviour): def test_ydl_args_parse_with_real_yt_dlp(tmp_path):
"""Install a stub yt_dlp whose download calls `behaviour(opts)`.""" """Every option songtrace passes must exist in the bundled yt-dlp."""
import sys import pytest
import types pytest.importorskip("yt_dlp")
from yt_dlp import parse_options
cfg = Config(whisper_url="http://x/v1", ytdlp_pot_provider_url="http://p:1",
ytdlp_cookies=tmp_path / "c.txt")
parse_options(acquire._ydl_args(cfg, tmp_path, "tv", "https://yt/x"))
class FakeYtDlp:
def __init__(self, version, behaviour, seen):
self.version, self._behaviour, self._seen = version, behaviour, seen
def run(self, args, log):
client = next((a for a in args if a.startswith("youtube:")), None)
self._seen.append((self.version, client))
self._behaviour(self.version, client)
def _fake_candidates(monkeypatch, behaviour, versions=("2026.09.01", "2026.08.19"),
refreshed=None):
"""Stub ytdlp.candidates; `refreshed` is what a refresh turns up."""
seen = [] seen = []
def candidates(cfg, log, refresh=False):
class FakeYoutubeDL: vs = refreshed if refresh and refreshed else versions
def __init__(self, opts): return [FakeYtDlp(v, behaviour, seen) for v in vs]
self.opts = opts monkeypatch.setattr(acquire.ytdlp, "candidates", candidates)
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def extract_info(self, url, download):
seen.append(self.opts["extractor_args"].get("youtube"))
behaviour(self.opts)
mod = types.ModuleType("yt_dlp")
mod.YoutubeDL = FakeYoutubeDL
mod.version = types.SimpleNamespace(__version__="2026.08.19")
monkeypatch.setitem(sys.modules, "yt_dlp", mod)
return seen return seen
def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch): def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch):
def behaviour(opts): from songtrace.ytdlp import YtDlpError
client = opts["extractor_args"].get("youtube") def behaviour(version, client):
if client is None: if client is None:
raise RuntimeError("HTTP Error 403: Forbidden") raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
(tmp_path / "source_video.webm").write_bytes(b"v") (tmp_path / "source_video.webm").write_bytes(b"v")
seen = _fake_yt_dlp(monkeypatch, behaviour) seen = _fake_candidates(monkeypatch, behaviour)
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"]) cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
logged = [] logged = []
video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg, video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg,
logged.append) logged.append)
assert video.name == "source_video.webm" assert video.name == "source_video.webm"
assert thumb is None assert thumb is None
assert seen == [None, {"player_client": ["tv"]}] assert seen == [("2026.09.01", None),
("2026.09.01", "youtube:player_client=tv")]
assert any("403" in m for m in logged) assert any("403" in m for m in logged)
def test_download_reports_version_and_clients_when_all_fail(tmp_path, monkeypatch): def test_download_falls_back_to_the_bundled_copy(tmp_path, monkeypatch):
from songtrace.ytdlp import YtDlpError
def behaviour(version, client):
if version == "2026.09.01":
raise YtDlpError("rejected songtrace's options")
(tmp_path / "source_video.webm").write_bytes(b"v")
seen = _fake_candidates(monkeypatch, behaviour)
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default"])
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
assert [v for v, _ in seen] == ["2026.09.01", "2026.08.19"]
def test_download_retries_with_a_release_found_after_failing(tmp_path, monkeypatch):
from songtrace.ytdlp import YtDlpError
def behaviour(version, client):
if version != "2026.10.01":
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
(tmp_path / "source_video.webm").write_bytes(b"v")
seen = _fake_candidates(monkeypatch, behaviour,
refreshed=("2026.10.01", "2026.08.19"))
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default"])
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
assert [v for v, _ in seen] == ["2026.09.01", "2026.08.19", "2026.10.01"]
def test_download_reports_versions_and_clients_when_all_fail(tmp_path, monkeypatch):
import pytest import pytest
from songtrace.pipeline import PipelineError from songtrace.pipeline import PipelineError
def behaviour(opts): from songtrace.ytdlp import YtDlpError
raise RuntimeError("HTTP Error 403: Forbidden") def behaviour(version, client):
_fake_yt_dlp(monkeypatch, behaviour) raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
seen = _fake_candidates(monkeypatch, behaviour)
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"]) cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
with pytest.raises(PipelineError) as e: with pytest.raises(PipelineError) as e:
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None) acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
assert "2026.08.19" in str(e.value) assert "2026.09.01, 2026.08.19" in str(e.value)
assert "default, tv" in str(e.value) assert "default, tv" in str(e.value)
assert "403" in str(e.value) assert "403" in str(e.value)
assert len(seen) == 4
def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch): def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch):
seen = _fake_yt_dlp( seen = _fake_candidates(
monkeypatch, lambda opts: (tmp_path / "source_video.mp4").write_bytes(b"v")) monkeypatch,
lambda v, c: (tmp_path / "source_video.mp4").write_bytes(b"v"))
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[]) cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[])
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None) acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
assert seen == [None] assert seen == [("2026.09.01", None)]
def test_ydl_log_drops_debug_lines():
logged = []
log = acquire._YdlLog(logged.append)
log.debug("[debug] internal detail")
log.debug("[youtube] Extracting URL")
log.warning("nsig extraction failed")
log.error("HTTP Error 403: Forbidden")
assert logged == ["[youtube] Extracting URL", "nsig extraction failed",
"HTTP Error 403: Forbidden"]
+139
View File
@@ -0,0 +1,139 @@
import os
import time
import pytest
from songtrace import ytdlp
from songtrace.config import Config
from songtrace.ytdlp import YtDlp, YtDlpError
def fake_package(path, version="2026.09.01", body=""):
"""A stand-in yt_dlp that answers --version and otherwise runs `body`."""
pkg = path / "yt_dlp"
pkg.mkdir(parents=True)
(pkg / "__init__.py").touch()
(pkg / "__main__.py").write_text(
"import sys\n"
f"if '--version' in sys.argv: print({version!r}); sys.exit()\n"
+ body)
return path
@pytest.fixture
def cache(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
monkeypatch.setattr(ytdlp, "bundled", lambda: YtDlp("2026.8.19"))
return ytdlp.cache_dir()
@pytest.fixture
def pypi(monkeypatch):
"""Stubs PyPI and pip: `latest` is the newest release, `installs` logs pip."""
state = {"latest": "2026.9.1", "installs": []}
monkeypatch.setattr(ytdlp, "_latest_release", lambda: state["latest"])
def pip_install(target, version):
state["installs"].append(version)
fake_package(target, version)
monkeypatch.setattr(ytdlp, "_pip_install", pip_install)
return state
def cfg(version="auto"):
return Config(whisper_url="http://x/v1", ytdlp_version=version)
def test_run_uses_the_install_on_its_path_and_logs_its_output(tmp_path):
path = fake_package(tmp_path, body="print('[youtube] hello', sys.argv[1:])\n")
logged = []
YtDlp("2026.09.01", path).run(["--", "https://yt/x"], logged.append)
assert logged == ["[youtube] hello ['--ignore-config', '--', 'https://yt/x']"]
def test_run_raises_the_error_line(tmp_path):
path = fake_package(tmp_path, body=(
"print('ERROR: HTTP Error 403: Forbidden', file=sys.stderr)\n"
"print('trailing noise')\nsys.exit(1)\n"))
with pytest.raises(YtDlpError, match="^ERROR: HTTP Error 403"):
YtDlp("2026.09.01", path).run([], lambda m: None)
def test_run_flags_a_usage_error_as_incompatible(tmp_path):
path = fake_package(tmp_path, body=(
"print('error: no such option: --retries', file=sys.stderr)\n"
"sys.exit(2)\n"))
with pytest.raises(YtDlpError, match="2026.09.01 rejected.*--retries"):
YtDlp("2026.09.01", path).run([], lambda m: None)
def test_auto_installs_a_newer_release_and_tries_it_first(cache, pypi):
found = ytdlp.candidates(cfg(), lambda m: None)
assert [y.version for y in found] == ["2026.9.1", "2026.8.19"]
assert found[0].path == cache / "2026.9.1"
assert not [p for p in cache.iterdir() if p.name.startswith(".tmp-")]
def test_auto_checks_pypi_once_a_day(cache, pypi):
ytdlp.candidates(cfg(), lambda m: None)
pypi["latest"] = "2026.10.1"
found = ytdlp.candidates(cfg(), lambda m: None)
assert found[0].version == "2026.9.1"
found = ytdlp.candidates(cfg(), lambda m: None, refresh=True)
assert found[0].version == "2026.10.1"
assert not (cache / "2026.9.1").exists()
old = time.time() - ytdlp.CHECK_INTERVAL - 1
os.utime(cache / ".checked", (old, old))
pypi["latest"] = "2026.11.1"
assert ytdlp.candidates(cfg(), lambda m: None)[0].version == "2026.11.1"
def test_auto_uses_only_bundled_when_it_is_newest(cache, pypi):
pypi["latest"] = "2026.8.19"
assert [y.version for y in ytdlp.candidates(cfg(), lambda m: None)] == [
"2026.8.19"]
assert pypi["installs"] == []
def test_auto_survives_pypi_being_unreachable(cache, pypi, monkeypatch):
ytdlp.candidates(cfg(), lambda m: None)
def down():
raise OSError("network unreachable")
monkeypatch.setattr(ytdlp, "_latest_release", down)
logged = []
found = ytdlp.candidates(cfg(), logged.append, refresh=True)
assert [y.version for y in found] == ["2026.9.1", "2026.8.19"]
assert any("network unreachable" in m for m in logged)
def test_an_install_that_fails_its_check_is_discarded(cache, pypi, monkeypatch):
monkeypatch.setattr(ytdlp, "_pip_install",
lambda target, v: fake_package(target, "2026.1.1"))
logged = []
assert [y.version for y in ytdlp.candidates(cfg(), logged.append)] == [
"2026.8.19"]
assert any("failed its check" in m for m in logged)
assert list(cache.iterdir()) == [cache / ".checked"]
def test_bundled_never_touches_pypi(cache, pypi):
pypi["latest"] = None
assert [y.version for y in ytdlp.candidates(cfg("bundled"), print)] == [
"2026.8.19"]
assert not cache.exists()
def test_pin_installs_that_release_even_if_older(cache, pypi):
found = ytdlp.candidates(cfg("2026.07.01"), lambda m: None)
assert [y.version for y in found] == ["2026.07.01", "2026.8.19"]
assert pypi["installs"] == ["2026.07.01"]
def test_pin_matching_bundled_installs_nothing(cache, pypi):
found = ytdlp.candidates(cfg("2026.08.19"), lambda m: None)
assert [y.version for y in found] == ["2026.8.19"]
assert pypi["installs"] == []
def test_pin_must_be_a_release_number(cache, pypi):
with pytest.raises(YtDlpError, match="ytdlp_version"):
ytdlp.candidates(cfg("latest; rm -rf /"), lambda m: None)