Keep yt-dlp current at runtime, with the bundled copy as fallback #12
@@ -74,7 +74,7 @@ See `config.example.toml` for every available field (`whisper_url`,
|
||||
`whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`,
|
||||
`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`,
|
||||
`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`,
|
||||
`ytdlp_pot_provider_url`).
|
||||
`ytdlp_pot_provider_url`, `ytdlp_version`).
|
||||
|
||||
`whisper_api` selects the wire format: `"openai"` (default) speaks the
|
||||
OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"`
|
||||
@@ -182,21 +182,24 @@ still wins.
|
||||
`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
|
||||
page but refused the media URL. In rough order of likelihood:
|
||||
|
||||
1. **yt-dlp is stale.** YouTube changes its defences every few weeks and the
|
||||
version baked into the image does not. Hotfix a running container, then
|
||||
rebuild so it sticks:
|
||||
1. **yt-dlp is stale.** YouTube changes its defences every few weeks, so
|
||||
songtrace keeps yt-dlp current itself. Once a day, and again whenever a
|
||||
download fails outright, it checks PyPI for a newer stable release,
|
||||
installs it under `$XDG_CACHE_HOME/songtrace/` (`/data/models/cache` in the
|
||||
container), and tries it before the copy installed with songtrace. That
|
||||
copy is never modified and is always the fallback, so a release that breaks
|
||||
compatibility costs a retry rather than the download. The job log names
|
||||
the yt-dlp version behind each attempt.
|
||||
|
||||
```sh
|
||||
docker compose exec songtrace pip install -U 'yt-dlp[default,curl-cffi,deno]'
|
||||
docker compose restart songtrace
|
||||
docker compose build --pull songtrace # permanent
|
||||
```
|
||||
`ytdlp_version` overrides this: `"bundled"` uses only the installed copy,
|
||||
and a release number such as `"2026.08.19"` pins that release instead of
|
||||
the newest.
|
||||
|
||||
2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine
|
||||
to solve YouTube's player challenges; without them it silently falls back to
|
||||
formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml`
|
||||
cover this. Check with
|
||||
`docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
|
||||
cover this, for both the installed copy and the ones fetched at runtime.
|
||||
Check with `docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
|
||||
header must show a `JS runtimes:` line that is not `none`.
|
||||
|
||||
3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get
|
||||
|
||||
@@ -18,3 +18,6 @@ bind_host = "0.0.0.0"
|
||||
# ytdlp_cookies = "/config/youtube-cookies.txt" # Netscape cookie jar
|
||||
# ytdlp_impersonate = "chrome" # TLS fingerprint to present
|
||||
# ytdlp_pot_provider_url = "http://bgutil-provider:4416"
|
||||
# Newer yt-dlp releases are fetched from PyPI at runtime and tried before the
|
||||
# bundled one; "bundled" turns that off, a release number pins one.
|
||||
# ytdlp_version = "auto"
|
||||
|
||||
@@ -24,6 +24,9 @@ class Config:
|
||||
ytdlp_cookies: Path | None = None
|
||||
ytdlp_impersonate: str | None = None
|
||||
ytdlp_pot_provider_url: str | None = None
|
||||
# "auto" keeps yt-dlp current from PyPI, "bundled" uses only the copy
|
||||
# installed with songtrace, and a release such as "2026.08.19" pins that.
|
||||
ytdlp_version: str = "auto"
|
||||
|
||||
|
||||
_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"}
|
||||
|
||||
@@ -2,6 +2,7 @@ import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from .. import ytdlp
|
||||
from ..config import Config
|
||||
from ..media import ffmpeg
|
||||
from ..pipeline import Ctx, PipelineError, Stage
|
||||
@@ -22,67 +23,50 @@ def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
|
||||
return video, thumbs[0] if thumbs else None
|
||||
|
||||
|
||||
class _YdlLog:
|
||||
"""Routes yt-dlp's own messages into the job log, minus progress noise."""
|
||||
|
||||
def __init__(self, log):
|
||||
self._log = log
|
||||
|
||||
def debug(self, msg: str) -> None:
|
||||
if not msg.startswith("[debug] "):
|
||||
self._log(msg)
|
||||
|
||||
def warning(self, msg: str) -> None:
|
||||
self._log(msg)
|
||||
|
||||
error = warning
|
||||
|
||||
|
||||
def _ydl_opts(cfg: Config, dest: Path, client: str, log) -> dict:
|
||||
extractor_args: dict[str, dict[str, list[str]]] = {}
|
||||
def _ydl_args(cfg: Config, dest: Path, client: str, url: str) -> list[str]:
|
||||
args = [
|
||||
"--output", str(dest / "source_video.%(ext)s"),
|
||||
"--format", "bestvideo*+bestaudio/best",
|
||||
"--write-thumbnail", "--no-playlist", "--no-progress",
|
||||
"--retries", "10", "--fragment-retries", "10",
|
||||
"--extractor-retries", "5",
|
||||
]
|
||||
if client != "default":
|
||||
extractor_args["youtube"] = {"player_client": [client]}
|
||||
args += ["--extractor-args", f"youtube:player_client={client}"]
|
||||
if cfg.ytdlp_pot_provider_url:
|
||||
extractor_args["youtubepot-bgutilhttp"] = {
|
||||
"base_url": [cfg.ytdlp_pot_provider_url]}
|
||||
opts = {
|
||||
"outtmpl": str(dest / "source_video.%(ext)s"),
|
||||
"format": "bestvideo*+bestaudio/best",
|
||||
"writethumbnail": True,
|
||||
"noplaylist": True,
|
||||
"noprogress": True,
|
||||
"logger": _YdlLog(log),
|
||||
"retries": 10,
|
||||
"fragment_retries": 10,
|
||||
"extractor_retries": 5,
|
||||
"extractor_args": extractor_args,
|
||||
}
|
||||
args += ["--extractor-args",
|
||||
f"youtubepot-bgutilhttp:base_url={cfg.ytdlp_pot_provider_url}"]
|
||||
if cfg.ytdlp_cookies:
|
||||
opts["cookiefile"] = str(cfg.ytdlp_cookies)
|
||||
args += ["--cookies", str(cfg.ytdlp_cookies)]
|
||||
if cfg.ytdlp_impersonate:
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
opts["impersonate"] = ImpersonateTarget.from_str(cfg.ytdlp_impersonate)
|
||||
return opts
|
||||
args += ["--impersonate", cfg.ytdlp_impersonate]
|
||||
return [*args, "--", url]
|
||||
|
||||
|
||||
def download_youtube(url: str, dest: Path, cfg: Config,
|
||||
log=print) -> tuple[Path, Path | None]:
|
||||
import yt_dlp
|
||||
clients = list(cfg.ytdlp_clients) or ["default"]
|
||||
tried: list[str] = []
|
||||
last: Exception | None = None
|
||||
for client in clients:
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(_ydl_opts(cfg, dest, client, log)) as ydl:
|
||||
ydl.extract_info(url, download=True)
|
||||
return _classify_downloads(dest)
|
||||
except Exception as e:
|
||||
last = e
|
||||
log(f"yt-dlp player_client={client} failed: {e}")
|
||||
# If every copy fails, look for a release newer than the daily check found.
|
||||
for refresh in (False, True):
|
||||
for ydl in ytdlp.candidates(cfg, log, refresh):
|
||||
if ydl.version in tried:
|
||||
continue
|
||||
tried.append(ydl.version)
|
||||
for client in clients:
|
||||
try:
|
||||
ydl.run(_ydl_args(cfg, dest, client, url), log)
|
||||
return _classify_downloads(dest)
|
||||
except PipelineError as e:
|
||||
last = e
|
||||
log(f"yt-dlp {ydl.version} player_client={client} "
|
||||
f"failed: {e}")
|
||||
raise PipelineError(
|
||||
f"yt-dlp {yt_dlp.version.__version__} could not download {url} after "
|
||||
f"trying player_client={', '.join(clients)}. Last error: {last}. "
|
||||
"YouTube blocks change often: upgrade yt-dlp, then see the yt-dlp "
|
||||
"options in config.toml (cookies, impersonation, PO token provider).")
|
||||
f"yt-dlp {', '.join(tried)} could not download {url} after trying "
|
||||
f"player_client={', '.join(clients)}. Last error: {last}. "
|
||||
"See the yt-dlp options in config.toml (cookies, impersonation, "
|
||||
"PO token provider).")
|
||||
|
||||
|
||||
def _run(ctx: Ctx) -> None:
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""The one place songtrace touches yt-dlp.
|
||||
|
||||
YouTube breaks old yt-dlp releases faster than songtrace is released, so
|
||||
besides the copy installed with songtrace (the bundled one), newer stable
|
||||
releases are fetched from PyPI into a cache dir at runtime and tried first,
|
||||
with the bundled copy as the fallback. yt-dlp runs in a subprocess and is
|
||||
driven through its command line, its most stable interface, so a cached
|
||||
release never mixes with anything songtrace has imported.
|
||||
"""
|
||||
import importlib.metadata
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
from .config import Config
|
||||
from .pipeline import PipelineError
|
||||
|
||||
# The extras are what make YouTube work; see pyproject.toml.
|
||||
REQUIREMENT = "yt-dlp[default,curl-cffi,deno]"
|
||||
CHECK_INTERVAL = 24 * 3600
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
class YtDlpError(PipelineError):
|
||||
pass
|
||||
|
||||
|
||||
def _key(version: str) -> tuple[int, ...]:
|
||||
return tuple(int(p) for p in version.split("."))
|
||||
|
||||
|
||||
def cache_dir() -> Path:
|
||||
base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
|
||||
# Installs hold compiled wheels, so they are only good for one Python.
|
||||
return Path(base) / "songtrace" / f"yt-dlp-py{sys.version_info[0]}{sys.version_info[1]}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class YtDlp:
|
||||
version: str
|
||||
path: Path | None = None # None for the bundled copy
|
||||
|
||||
def _env(self) -> dict[str, str]:
|
||||
env = dict(os.environ)
|
||||
if self.path:
|
||||
env["PYTHONPATH"] = os.pathsep.join(
|
||||
filter(None, [str(self.path), env.get("PYTHONPATH")]))
|
||||
env["PATH"] = os.pathsep.join(
|
||||
[str(self.path / "bin"), env.get("PATH", "")])
|
||||
return env
|
||||
|
||||
def run(self, args: list[str], log) -> None:
|
||||
"""Runs yt-dlp with `args`, passing each line it prints to `log`."""
|
||||
proc = subprocess.Popen(
|
||||
[sys.executable, "-P", "-m", "yt_dlp", "--ignore-config", *args],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
||||
errors="replace", env=self._env())
|
||||
last = error = ""
|
||||
for line in proc.stdout:
|
||||
last = line.rstrip()
|
||||
log(last)
|
||||
if last.startswith("ERROR:"):
|
||||
error = last
|
||||
if proc.wait() == 0:
|
||||
return
|
||||
if proc.returncode == 2:
|
||||
raise YtDlpError(f"yt-dlp {self.version} rejected songtrace's "
|
||||
f"options, so it may be incompatible: {last}")
|
||||
raise YtDlpError(error or f"yt-dlp exited with status {proc.returncode}")
|
||||
|
||||
|
||||
def bundled() -> YtDlp:
|
||||
return YtDlp(importlib.metadata.version("yt-dlp"))
|
||||
|
||||
|
||||
def _latest_release() -> str:
|
||||
r = httpx.get("https://pypi.org/pypi/yt-dlp/json", timeout=15,
|
||||
follow_redirects=True)
|
||||
r.raise_for_status()
|
||||
return r.json()["info"]["version"]
|
||||
|
||||
|
||||
def _pip_install(target: Path, version: str) -> None:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "install", "--quiet", "--no-input",
|
||||
"--disable-pip-version-check", "--target", str(target),
|
||||
f"{REQUIREMENT}=={version}"],
|
||||
capture_output=True, text=True)
|
||||
if proc.returncode:
|
||||
raise YtDlpError(f"pip could not install yt-dlp {version}: "
|
||||
f"{proc.stderr.strip()[-500:]}")
|
||||
|
||||
|
||||
def _install(version: str, log) -> YtDlp:
|
||||
"""Installs `version` into the cache unless it is already there.
|
||||
|
||||
The install is checked to run before it is moved into place, so a
|
||||
directory in the cache is always a usable yt-dlp."""
|
||||
dest = cache_dir() / version
|
||||
if dest.is_dir():
|
||||
return YtDlp(version, dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
log(f"installing yt-dlp {version}")
|
||||
tmp = Path(tempfile.mkdtemp(dir=dest.parent, prefix=".tmp-"))
|
||||
try:
|
||||
_pip_install(tmp, version)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "-P", "-m", "yt_dlp", "--version"],
|
||||
capture_output=True, text=True, env=YtDlp(version, tmp)._env())
|
||||
got = proc.stdout.strip()
|
||||
if proc.returncode or _key(got) != _key(version):
|
||||
raise YtDlpError(f"yt-dlp {version} failed its check after install: "
|
||||
f"{(got or proc.stderr.strip())[-500:]}")
|
||||
try:
|
||||
tmp.rename(dest)
|
||||
except OSError:
|
||||
if not dest.is_dir():
|
||||
raise
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
return YtDlp(version, dest)
|
||||
|
||||
|
||||
def _cached() -> list[YtDlp]:
|
||||
root = cache_dir()
|
||||
if not root.is_dir():
|
||||
return []
|
||||
found = [YtDlp(p.name, p) for p in root.iterdir()
|
||||
if p.is_dir() and re.fullmatch(r"\d+(\.\d+)*", p.name)]
|
||||
return sorted(found, key=lambda y: _key(y.version))
|
||||
|
||||
|
||||
def _newest(base: YtDlp, refresh: bool, log) -> YtDlp | None:
|
||||
"""The newest cached release newer than `base`, checking PyPI for a newer
|
||||
one first if that has not been done for a day or `refresh` is set."""
|
||||
marker = cache_dir() / ".checked"
|
||||
if (refresh or not marker.exists()
|
||||
or time.time() - marker.stat().st_mtime > CHECK_INTERVAL):
|
||||
# Recorded even on failure, so a broken release is not reinstalled
|
||||
# for every job; a failed download refreshes regardless.
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
marker.touch()
|
||||
try:
|
||||
latest = _latest_release()
|
||||
if _key(latest) > _key(base.version):
|
||||
_install(latest, log)
|
||||
except Exception as e:
|
||||
log(f"could not update yt-dlp: {e}")
|
||||
newer = [y for y in _cached() if _key(y.version) > _key(base.version)]
|
||||
for old in newer[:-1]:
|
||||
shutil.rmtree(old.path, ignore_errors=True)
|
||||
return newer[-1] if newer else None
|
||||
|
||||
|
||||
def candidates(cfg: Config, log, refresh: bool = False) -> list[YtDlp]:
|
||||
"""The yt-dlp copies to try, in order: the newest release (or the one
|
||||
`cfg.ytdlp_version` pins), then the bundled copy."""
|
||||
base = bundled()
|
||||
want = cfg.ytdlp_version
|
||||
if want == "bundled":
|
||||
return [base]
|
||||
with _lock:
|
||||
if want == "auto":
|
||||
first = _newest(base, refresh, log)
|
||||
elif not re.fullmatch(r"\d+(\.\d+)*", want):
|
||||
raise YtDlpError(f"ytdlp_version must be \"auto\", \"bundled\" or a "
|
||||
f"release such as \"2026.08.19\", not {want!r}")
|
||||
elif _key(want) == _key(base.version):
|
||||
first = None
|
||||
else:
|
||||
try:
|
||||
first = _install(want, log)
|
||||
except Exception as e:
|
||||
log(f"could not install the pinned yt-dlp {want}: {e}")
|
||||
first = None
|
||||
return [first, base] if first else [base]
|
||||
+79
-54
@@ -91,87 +91,112 @@ def test_classify_ignores_partials_left_by_a_failed_attempt(tmp_path):
|
||||
video, _ = acquire._classify_downloads(tmp_path)
|
||||
assert video.name == "source_video.webm"
|
||||
|
||||
def test_ydl_opts_default_client_adds_no_player_client(tmp_path):
|
||||
def test_ydl_args_default_client_adds_no_player_client(tmp_path):
|
||||
cfg = Config(whisper_url="http://x/v1")
|
||||
opts = acquire._ydl_opts(cfg, tmp_path, "default", print)
|
||||
assert "youtube" not in opts["extractor_args"]
|
||||
assert opts["retries"] and opts["fragment_retries"] and opts["extractor_retries"]
|
||||
args = acquire._ydl_args(cfg, tmp_path, "default", "https://yt/x")
|
||||
assert "--extractor-args" not in args
|
||||
assert "--retries" in args and "--fragment-retries" in args
|
||||
assert args[-2:] == ["--", "https://yt/x"]
|
||||
|
||||
def test_ydl_opts_named_client_and_pot_provider(tmp_path):
|
||||
def test_ydl_args_named_client_pot_provider_and_cookies(tmp_path):
|
||||
cfg = Config(whisper_url="http://x/v1",
|
||||
ytdlp_pot_provider_url="http://pot:4416",
|
||||
ytdlp_cookies=tmp_path / "cookies.txt")
|
||||
opts = acquire._ydl_opts(cfg, tmp_path, "tv", print)
|
||||
assert opts["extractor_args"]["youtube"] == {"player_client": ["tv"]}
|
||||
assert opts["extractor_args"]["youtubepot-bgutilhttp"] == {
|
||||
"base_url": ["http://pot:4416"]}
|
||||
assert opts["cookiefile"] == str(tmp_path / "cookies.txt")
|
||||
ytdlp_cookies=tmp_path / "cookies.txt",
|
||||
ytdlp_impersonate="chrome")
|
||||
args = acquire._ydl_args(cfg, tmp_path, "tv", "https://yt/x")
|
||||
joined = " ".join(args)
|
||||
assert "--extractor-args youtube:player_client=tv" in joined
|
||||
assert ("--extractor-args youtubepot-bgutilhttp:base_url=http://pot:4416"
|
||||
in joined)
|
||||
assert f"--cookies {tmp_path / 'cookies.txt'}" in joined
|
||||
assert "--impersonate chrome" in joined
|
||||
|
||||
def _fake_yt_dlp(monkeypatch, behaviour):
|
||||
"""Install a stub yt_dlp whose download calls `behaviour(opts)`."""
|
||||
import sys
|
||||
import types
|
||||
def test_ydl_args_parse_with_real_yt_dlp(tmp_path):
|
||||
"""Every option songtrace passes must exist in the bundled yt-dlp."""
|
||||
import pytest
|
||||
pytest.importorskip("yt_dlp")
|
||||
from yt_dlp import parse_options
|
||||
cfg = Config(whisper_url="http://x/v1", ytdlp_pot_provider_url="http://p:1",
|
||||
ytdlp_cookies=tmp_path / "c.txt")
|
||||
parse_options(acquire._ydl_args(cfg, tmp_path, "tv", "https://yt/x"))
|
||||
|
||||
class FakeYtDlp:
|
||||
def __init__(self, version, behaviour, seen):
|
||||
self.version, self._behaviour, self._seen = version, behaviour, seen
|
||||
def run(self, args, log):
|
||||
client = next((a for a in args if a.startswith("youtube:")), None)
|
||||
self._seen.append((self.version, client))
|
||||
self._behaviour(self.version, client)
|
||||
|
||||
def _fake_candidates(monkeypatch, behaviour, versions=("2026.09.01", "2026.08.19"),
|
||||
refreshed=None):
|
||||
"""Stub ytdlp.candidates; `refreshed` is what a refresh turns up."""
|
||||
seen = []
|
||||
|
||||
class FakeYoutubeDL:
|
||||
def __init__(self, opts):
|
||||
self.opts = opts
|
||||
def __enter__(self):
|
||||
return self
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
def extract_info(self, url, download):
|
||||
seen.append(self.opts["extractor_args"].get("youtube"))
|
||||
behaviour(self.opts)
|
||||
|
||||
mod = types.ModuleType("yt_dlp")
|
||||
mod.YoutubeDL = FakeYoutubeDL
|
||||
mod.version = types.SimpleNamespace(__version__="2026.08.19")
|
||||
monkeypatch.setitem(sys.modules, "yt_dlp", mod)
|
||||
def candidates(cfg, log, refresh=False):
|
||||
vs = refreshed if refresh and refreshed else versions
|
||||
return [FakeYtDlp(v, behaviour, seen) for v in vs]
|
||||
monkeypatch.setattr(acquire.ytdlp, "candidates", candidates)
|
||||
return seen
|
||||
|
||||
def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch):
|
||||
def behaviour(opts):
|
||||
client = opts["extractor_args"].get("youtube")
|
||||
from songtrace.ytdlp import YtDlpError
|
||||
def behaviour(version, client):
|
||||
if client is None:
|
||||
raise RuntimeError("HTTP Error 403: Forbidden")
|
||||
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
|
||||
(tmp_path / "source_video.webm").write_bytes(b"v")
|
||||
seen = _fake_yt_dlp(monkeypatch, behaviour)
|
||||
seen = _fake_candidates(monkeypatch, behaviour)
|
||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
|
||||
logged = []
|
||||
video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg,
|
||||
logged.append)
|
||||
assert video.name == "source_video.webm"
|
||||
assert thumb is None
|
||||
assert seen == [None, {"player_client": ["tv"]}]
|
||||
assert seen == [("2026.09.01", None),
|
||||
("2026.09.01", "youtube:player_client=tv")]
|
||||
assert any("403" in m for m in logged)
|
||||
|
||||
def test_download_reports_version_and_clients_when_all_fail(tmp_path, monkeypatch):
|
||||
def test_download_falls_back_to_the_bundled_copy(tmp_path, monkeypatch):
|
||||
from songtrace.ytdlp import YtDlpError
|
||||
def behaviour(version, client):
|
||||
if version == "2026.09.01":
|
||||
raise YtDlpError("rejected songtrace's options")
|
||||
(tmp_path / "source_video.webm").write_bytes(b"v")
|
||||
seen = _fake_candidates(monkeypatch, behaviour)
|
||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default"])
|
||||
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||
assert [v for v, _ in seen] == ["2026.09.01", "2026.08.19"]
|
||||
|
||||
def test_download_retries_with_a_release_found_after_failing(tmp_path, monkeypatch):
|
||||
from songtrace.ytdlp import YtDlpError
|
||||
def behaviour(version, client):
|
||||
if version != "2026.10.01":
|
||||
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
|
||||
(tmp_path / "source_video.webm").write_bytes(b"v")
|
||||
seen = _fake_candidates(monkeypatch, behaviour,
|
||||
refreshed=("2026.10.01", "2026.08.19"))
|
||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default"])
|
||||
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||
assert [v for v, _ in seen] == ["2026.09.01", "2026.08.19", "2026.10.01"]
|
||||
|
||||
def test_download_reports_versions_and_clients_when_all_fail(tmp_path, monkeypatch):
|
||||
import pytest
|
||||
from songtrace.pipeline import PipelineError
|
||||
def behaviour(opts):
|
||||
raise RuntimeError("HTTP Error 403: Forbidden")
|
||||
_fake_yt_dlp(monkeypatch, behaviour)
|
||||
from songtrace.ytdlp import YtDlpError
|
||||
def behaviour(version, client):
|
||||
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
|
||||
seen = _fake_candidates(monkeypatch, behaviour)
|
||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
|
||||
with pytest.raises(PipelineError) as e:
|
||||
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||
assert "2026.08.19" in str(e.value)
|
||||
assert "2026.09.01, 2026.08.19" in str(e.value)
|
||||
assert "default, tv" in str(e.value)
|
||||
assert "403" in str(e.value)
|
||||
assert len(seen) == 4
|
||||
|
||||
def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch):
|
||||
seen = _fake_yt_dlp(
|
||||
monkeypatch, lambda opts: (tmp_path / "source_video.mp4").write_bytes(b"v"))
|
||||
seen = _fake_candidates(
|
||||
monkeypatch,
|
||||
lambda v, c: (tmp_path / "source_video.mp4").write_bytes(b"v"))
|
||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[])
|
||||
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||
assert seen == [None]
|
||||
|
||||
def test_ydl_log_drops_debug_lines():
|
||||
logged = []
|
||||
log = acquire._YdlLog(logged.append)
|
||||
log.debug("[debug] internal detail")
|
||||
log.debug("[youtube] Extracting URL")
|
||||
log.warning("nsig extraction failed")
|
||||
log.error("HTTP Error 403: Forbidden")
|
||||
assert logged == ["[youtube] Extracting URL", "nsig extraction failed",
|
||||
"HTTP Error 403: Forbidden"]
|
||||
assert seen == [("2026.09.01", None)]
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from songtrace import ytdlp
|
||||
from songtrace.config import Config
|
||||
from songtrace.ytdlp import YtDlp, YtDlpError
|
||||
|
||||
|
||||
def fake_package(path, version="2026.09.01", body=""):
|
||||
"""A stand-in yt_dlp that answers --version and otherwise runs `body`."""
|
||||
pkg = path / "yt_dlp"
|
||||
pkg.mkdir(parents=True)
|
||||
(pkg / "__init__.py").touch()
|
||||
(pkg / "__main__.py").write_text(
|
||||
"import sys\n"
|
||||
f"if '--version' in sys.argv: print({version!r}); sys.exit()\n"
|
||||
+ body)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
|
||||
monkeypatch.setattr(ytdlp, "bundled", lambda: YtDlp("2026.8.19"))
|
||||
return ytdlp.cache_dir()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pypi(monkeypatch):
|
||||
"""Stubs PyPI and pip: `latest` is the newest release, `installs` logs pip."""
|
||||
state = {"latest": "2026.9.1", "installs": []}
|
||||
monkeypatch.setattr(ytdlp, "_latest_release", lambda: state["latest"])
|
||||
def pip_install(target, version):
|
||||
state["installs"].append(version)
|
||||
fake_package(target, version)
|
||||
monkeypatch.setattr(ytdlp, "_pip_install", pip_install)
|
||||
return state
|
||||
|
||||
|
||||
def cfg(version="auto"):
|
||||
return Config(whisper_url="http://x/v1", ytdlp_version=version)
|
||||
|
||||
|
||||
def test_run_uses_the_install_on_its_path_and_logs_its_output(tmp_path):
|
||||
path = fake_package(tmp_path, body="print('[youtube] hello', sys.argv[1:])\n")
|
||||
logged = []
|
||||
YtDlp("2026.09.01", path).run(["--", "https://yt/x"], logged.append)
|
||||
assert logged == ["[youtube] hello ['--ignore-config', '--', 'https://yt/x']"]
|
||||
|
||||
|
||||
def test_run_raises_the_error_line(tmp_path):
|
||||
path = fake_package(tmp_path, body=(
|
||||
"print('ERROR: HTTP Error 403: Forbidden', file=sys.stderr)\n"
|
||||
"print('trailing noise')\nsys.exit(1)\n"))
|
||||
with pytest.raises(YtDlpError, match="^ERROR: HTTP Error 403"):
|
||||
YtDlp("2026.09.01", path).run([], lambda m: None)
|
||||
|
||||
|
||||
def test_run_flags_a_usage_error_as_incompatible(tmp_path):
|
||||
path = fake_package(tmp_path, body=(
|
||||
"print('error: no such option: --retries', file=sys.stderr)\n"
|
||||
"sys.exit(2)\n"))
|
||||
with pytest.raises(YtDlpError, match="2026.09.01 rejected.*--retries"):
|
||||
YtDlp("2026.09.01", path).run([], lambda m: None)
|
||||
|
||||
|
||||
def test_auto_installs_a_newer_release_and_tries_it_first(cache, pypi):
|
||||
found = ytdlp.candidates(cfg(), lambda m: None)
|
||||
assert [y.version for y in found] == ["2026.9.1", "2026.8.19"]
|
||||
assert found[0].path == cache / "2026.9.1"
|
||||
assert not [p for p in cache.iterdir() if p.name.startswith(".tmp-")]
|
||||
|
||||
|
||||
def test_auto_checks_pypi_once_a_day(cache, pypi):
|
||||
ytdlp.candidates(cfg(), lambda m: None)
|
||||
pypi["latest"] = "2026.10.1"
|
||||
found = ytdlp.candidates(cfg(), lambda m: None)
|
||||
assert found[0].version == "2026.9.1"
|
||||
found = ytdlp.candidates(cfg(), lambda m: None, refresh=True)
|
||||
assert found[0].version == "2026.10.1"
|
||||
assert not (cache / "2026.9.1").exists()
|
||||
old = time.time() - ytdlp.CHECK_INTERVAL - 1
|
||||
os.utime(cache / ".checked", (old, old))
|
||||
pypi["latest"] = "2026.11.1"
|
||||
assert ytdlp.candidates(cfg(), lambda m: None)[0].version == "2026.11.1"
|
||||
|
||||
|
||||
def test_auto_uses_only_bundled_when_it_is_newest(cache, pypi):
|
||||
pypi["latest"] = "2026.8.19"
|
||||
assert [y.version for y in ytdlp.candidates(cfg(), lambda m: None)] == [
|
||||
"2026.8.19"]
|
||||
assert pypi["installs"] == []
|
||||
|
||||
|
||||
def test_auto_survives_pypi_being_unreachable(cache, pypi, monkeypatch):
|
||||
ytdlp.candidates(cfg(), lambda m: None)
|
||||
def down():
|
||||
raise OSError("network unreachable")
|
||||
monkeypatch.setattr(ytdlp, "_latest_release", down)
|
||||
logged = []
|
||||
found = ytdlp.candidates(cfg(), logged.append, refresh=True)
|
||||
assert [y.version for y in found] == ["2026.9.1", "2026.8.19"]
|
||||
assert any("network unreachable" in m for m in logged)
|
||||
|
||||
|
||||
def test_an_install_that_fails_its_check_is_discarded(cache, pypi, monkeypatch):
|
||||
monkeypatch.setattr(ytdlp, "_pip_install",
|
||||
lambda target, v: fake_package(target, "2026.1.1"))
|
||||
logged = []
|
||||
assert [y.version for y in ytdlp.candidates(cfg(), logged.append)] == [
|
||||
"2026.8.19"]
|
||||
assert any("failed its check" in m for m in logged)
|
||||
assert list(cache.iterdir()) == [cache / ".checked"]
|
||||
|
||||
|
||||
def test_bundled_never_touches_pypi(cache, pypi):
|
||||
pypi["latest"] = None
|
||||
assert [y.version for y in ytdlp.candidates(cfg("bundled"), print)] == [
|
||||
"2026.8.19"]
|
||||
assert not cache.exists()
|
||||
|
||||
|
||||
def test_pin_installs_that_release_even_if_older(cache, pypi):
|
||||
found = ytdlp.candidates(cfg("2026.07.01"), lambda m: None)
|
||||
assert [y.version for y in found] == ["2026.07.01", "2026.8.19"]
|
||||
assert pypi["installs"] == ["2026.07.01"]
|
||||
|
||||
|
||||
def test_pin_matching_bundled_installs_nothing(cache, pypi):
|
||||
found = ytdlp.candidates(cfg("2026.08.19"), lambda m: None)
|
||||
assert [y.version for y in found] == ["2026.8.19"]
|
||||
assert pypi["installs"] == []
|
||||
|
||||
|
||||
def test_pin_must_be_a_release_number(cache, pypi):
|
||||
with pytest.raises(YtDlpError, match="ytdlp_version"):
|
||||
ytdlp.candidates(cfg("latest; rm -rf /"), lambda m: None)
|
||||
Reference in New Issue
Block a user