Keep yt-dlp current at runtime, with the bundled copy as fallback #12
@@ -74,7 +74,7 @@ See `config.example.toml` for every available field (`whisper_url`,
|
|||||||
`whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`,
|
`whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`,
|
||||||
`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`,
|
`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`,
|
||||||
`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`,
|
`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`,
|
||||||
`ytdlp_pot_provider_url`).
|
`ytdlp_pot_provider_url`, `ytdlp_version`).
|
||||||
|
|
||||||
`whisper_api` selects the wire format: `"openai"` (default) speaks the
|
`whisper_api` selects the wire format: `"openai"` (default) speaks the
|
||||||
OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"`
|
OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"`
|
||||||
@@ -182,21 +182,24 @@ still wins.
|
|||||||
`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
|
`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
|
||||||
page but refused the media URL. In rough order of likelihood:
|
page but refused the media URL. In rough order of likelihood:
|
||||||
|
|
||||||
1. **yt-dlp is stale.** YouTube changes its defences every few weeks and the
|
1. **yt-dlp is stale.** YouTube changes its defences every few weeks, so
|
||||||
version baked into the image does not. Hotfix a running container, then
|
songtrace keeps yt-dlp current itself. Once a day, and again whenever a
|
||||||
rebuild so it sticks:
|
download fails outright, it checks PyPI for a newer stable release,
|
||||||
|
installs it under `$XDG_CACHE_HOME/songtrace/` (`/data/models/cache` in the
|
||||||
|
container), and tries it before the copy installed with songtrace. That
|
||||||
|
copy is never modified and is always the fallback, so a release that breaks
|
||||||
|
compatibility costs a retry rather than the download. The job log names
|
||||||
|
the yt-dlp version behind each attempt.
|
||||||
|
|
||||||
```sh
|
`ytdlp_version` overrides this: `"bundled"` uses only the installed copy,
|
||||||
docker compose exec songtrace pip install -U 'yt-dlp[default,curl-cffi,deno]'
|
and a release number such as `"2026.08.19"` pins that release instead of
|
||||||
docker compose restart songtrace
|
the newest.
|
||||||
docker compose build --pull songtrace # permanent
|
|
||||||
```
|
|
||||||
|
|
||||||
2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine
|
2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine
|
||||||
to solve YouTube's player challenges; without them it silently falls back to
|
to solve YouTube's player challenges; without them it silently falls back to
|
||||||
formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml`
|
formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml`
|
||||||
cover this. Check with
|
cover this, for both the installed copy and the ones fetched at runtime.
|
||||||
`docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
|
Check with `docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
|
||||||
header must show a `JS runtimes:` line that is not `none`.
|
header must show a `JS runtimes:` line that is not `none`.
|
||||||
|
|
||||||
3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get
|
3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get
|
||||||
|
|||||||
@@ -18,3 +18,6 @@ bind_host = "0.0.0.0"
|
|||||||
# ytdlp_cookies = "/config/youtube-cookies.txt" # Netscape cookie jar
|
# ytdlp_cookies = "/config/youtube-cookies.txt" # Netscape cookie jar
|
||||||
# ytdlp_impersonate = "chrome" # TLS fingerprint to present
|
# ytdlp_impersonate = "chrome" # TLS fingerprint to present
|
||||||
# ytdlp_pot_provider_url = "http://bgutil-provider:4416"
|
# ytdlp_pot_provider_url = "http://bgutil-provider:4416"
|
||||||
|
# Newer yt-dlp releases are fetched from PyPI at runtime and tried before the
|
||||||
|
# bundled one; "bundled" turns that off, a release number pins one.
|
||||||
|
# ytdlp_version = "auto"
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ class Config:
|
|||||||
ytdlp_cookies: Path | None = None
|
ytdlp_cookies: Path | None = None
|
||||||
ytdlp_impersonate: str | None = None
|
ytdlp_impersonate: str | None = None
|
||||||
ytdlp_pot_provider_url: str | None = None
|
ytdlp_pot_provider_url: str | None = None
|
||||||
|
# "auto" keeps yt-dlp current from PyPI, "bundled" uses only the copy
|
||||||
|
# installed with songtrace, and a release such as "2026.08.19" pins that.
|
||||||
|
ytdlp_version: str = "auto"
|
||||||
|
|
||||||
|
|
||||||
_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"}
|
_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import json
|
|||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .. import ytdlp
|
||||||
from ..config import Config
|
from ..config import Config
|
||||||
from ..media import ffmpeg
|
from ..media import ffmpeg
|
||||||
from ..pipeline import Ctx, PipelineError, Stage
|
from ..pipeline import Ctx, PipelineError, Stage
|
||||||
@@ -22,67 +23,50 @@ def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
|
|||||||
return video, thumbs[0] if thumbs else None
|
return video, thumbs[0] if thumbs else None
|
||||||
|
|
||||||
|
|
||||||
class _YdlLog:
|
def _ydl_args(cfg: Config, dest: Path, client: str, url: str) -> list[str]:
|
||||||
"""Routes yt-dlp's own messages into the job log, minus progress noise."""
|
args = [
|
||||||
|
"--output", str(dest / "source_video.%(ext)s"),
|
||||||
def __init__(self, log):
|
"--format", "bestvideo*+bestaudio/best",
|
||||||
self._log = log
|
"--write-thumbnail", "--no-playlist", "--no-progress",
|
||||||
|
"--retries", "10", "--fragment-retries", "10",
|
||||||
def debug(self, msg: str) -> None:
|
"--extractor-retries", "5",
|
||||||
if not msg.startswith("[debug] "):
|
]
|
||||||
self._log(msg)
|
|
||||||
|
|
||||||
def warning(self, msg: str) -> None:
|
|
||||||
self._log(msg)
|
|
||||||
|
|
||||||
error = warning
|
|
||||||
|
|
||||||
|
|
||||||
def _ydl_opts(cfg: Config, dest: Path, client: str, log) -> dict:
|
|
||||||
extractor_args: dict[str, dict[str, list[str]]] = {}
|
|
||||||
if client != "default":
|
if client != "default":
|
||||||
extractor_args["youtube"] = {"player_client": [client]}
|
args += ["--extractor-args", f"youtube:player_client={client}"]
|
||||||
if cfg.ytdlp_pot_provider_url:
|
if cfg.ytdlp_pot_provider_url:
|
||||||
extractor_args["youtubepot-bgutilhttp"] = {
|
args += ["--extractor-args",
|
||||||
"base_url": [cfg.ytdlp_pot_provider_url]}
|
f"youtubepot-bgutilhttp:base_url={cfg.ytdlp_pot_provider_url}"]
|
||||||
opts = {
|
|
||||||
"outtmpl": str(dest / "source_video.%(ext)s"),
|
|
||||||
"format": "bestvideo*+bestaudio/best",
|
|
||||||
"writethumbnail": True,
|
|
||||||
"noplaylist": True,
|
|
||||||
"noprogress": True,
|
|
||||||
"logger": _YdlLog(log),
|
|
||||||
"retries": 10,
|
|
||||||
"fragment_retries": 10,
|
|
||||||
"extractor_retries": 5,
|
|
||||||
"extractor_args": extractor_args,
|
|
||||||
}
|
|
||||||
if cfg.ytdlp_cookies:
|
if cfg.ytdlp_cookies:
|
||||||
opts["cookiefile"] = str(cfg.ytdlp_cookies)
|
args += ["--cookies", str(cfg.ytdlp_cookies)]
|
||||||
if cfg.ytdlp_impersonate:
|
if cfg.ytdlp_impersonate:
|
||||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
args += ["--impersonate", cfg.ytdlp_impersonate]
|
||||||
opts["impersonate"] = ImpersonateTarget.from_str(cfg.ytdlp_impersonate)
|
return [*args, "--", url]
|
||||||
return opts
|
|
||||||
|
|
||||||
|
|
||||||
def download_youtube(url: str, dest: Path, cfg: Config,
|
def download_youtube(url: str, dest: Path, cfg: Config,
|
||||||
log=print) -> tuple[Path, Path | None]:
|
log=print) -> tuple[Path, Path | None]:
|
||||||
import yt_dlp
|
|
||||||
clients = list(cfg.ytdlp_clients) or ["default"]
|
clients = list(cfg.ytdlp_clients) or ["default"]
|
||||||
|
tried: list[str] = []
|
||||||
last: Exception | None = None
|
last: Exception | None = None
|
||||||
for client in clients:
|
# If every copy fails, look for a release newer than the daily check found.
|
||||||
try:
|
for refresh in (False, True):
|
||||||
with yt_dlp.YoutubeDL(_ydl_opts(cfg, dest, client, log)) as ydl:
|
for ydl in ytdlp.candidates(cfg, log, refresh):
|
||||||
ydl.extract_info(url, download=True)
|
if ydl.version in tried:
|
||||||
return _classify_downloads(dest)
|
continue
|
||||||
except Exception as e:
|
tried.append(ydl.version)
|
||||||
last = e
|
for client in clients:
|
||||||
log(f"yt-dlp player_client={client} failed: {e}")
|
try:
|
||||||
|
ydl.run(_ydl_args(cfg, dest, client, url), log)
|
||||||
|
return _classify_downloads(dest)
|
||||||
|
except PipelineError as e:
|
||||||
|
last = e
|
||||||
|
log(f"yt-dlp {ydl.version} player_client={client} "
|
||||||
|
f"failed: {e}")
|
||||||
raise PipelineError(
|
raise PipelineError(
|
||||||
f"yt-dlp {yt_dlp.version.__version__} could not download {url} after "
|
f"yt-dlp {', '.join(tried)} could not download {url} after trying "
|
||||||
f"trying player_client={', '.join(clients)}. Last error: {last}. "
|
f"player_client={', '.join(clients)}. Last error: {last}. "
|
||||||
"YouTube blocks change often: upgrade yt-dlp, then see the yt-dlp "
|
"See the yt-dlp options in config.toml (cookies, impersonation, "
|
||||||
"options in config.toml (cookies, impersonation, PO token provider).")
|
"PO token provider).")
|
||||||
|
|
||||||
|
|
||||||
def _run(ctx: Ctx) -> None:
|
def _run(ctx: Ctx) -> None:
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
"""The one place songtrace touches yt-dlp.
|
||||||
|
|
||||||
|
YouTube breaks old yt-dlp releases faster than songtrace is released, so
|
||||||
|
besides the copy installed with songtrace (the bundled one), newer stable
|
||||||
|
releases are fetched from PyPI into a cache dir at runtime and tried first,
|
||||||
|
with the bundled copy as the fallback. yt-dlp runs in a subprocess and is
|
||||||
|
driven through its command line, its most stable interface, so a cached
|
||||||
|
release never mixes with anything songtrace has imported.
|
||||||
|
"""
|
||||||
|
import importlib.metadata
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .config import Config
|
||||||
|
from .pipeline import PipelineError
|
||||||
|
|
||||||
|
# The extras are what make YouTube work; see pyproject.toml.
|
||||||
|
REQUIREMENT = "yt-dlp[default,curl-cffi,deno]"
|
||||||
|
CHECK_INTERVAL = 24 * 3600
|
||||||
|
_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
class YtDlpError(PipelineError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _key(version: str) -> tuple[int, ...]:
|
||||||
|
return tuple(int(p) for p in version.split("."))
|
||||||
|
|
||||||
|
|
||||||
|
def cache_dir() -> Path:
|
||||||
|
base = os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache"
|
||||||
|
# Installs hold compiled wheels, so they are only good for one Python.
|
||||||
|
return Path(base) / "songtrace" / f"yt-dlp-py{sys.version_info[0]}{sys.version_info[1]}"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class YtDlp:
|
||||||
|
version: str
|
||||||
|
path: Path | None = None # None for the bundled copy
|
||||||
|
|
||||||
|
def _env(self) -> dict[str, str]:
|
||||||
|
env = dict(os.environ)
|
||||||
|
if self.path:
|
||||||
|
env["PYTHONPATH"] = os.pathsep.join(
|
||||||
|
filter(None, [str(self.path), env.get("PYTHONPATH")]))
|
||||||
|
env["PATH"] = os.pathsep.join(
|
||||||
|
[str(self.path / "bin"), env.get("PATH", "")])
|
||||||
|
return env
|
||||||
|
|
||||||
|
def run(self, args: list[str], log) -> None:
|
||||||
|
"""Runs yt-dlp with `args`, passing each line it prints to `log`."""
|
||||||
|
proc = subprocess.Popen(
|
||||||
|
[sys.executable, "-P", "-m", "yt_dlp", "--ignore-config", *args],
|
||||||
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
||||||
|
errors="replace", env=self._env())
|
||||||
|
last = error = ""
|
||||||
|
for line in proc.stdout:
|
||||||
|
last = line.rstrip()
|
||||||
|
log(last)
|
||||||
|
if last.startswith("ERROR:"):
|
||||||
|
error = last
|
||||||
|
if proc.wait() == 0:
|
||||||
|
return
|
||||||
|
if proc.returncode == 2:
|
||||||
|
raise YtDlpError(f"yt-dlp {self.version} rejected songtrace's "
|
||||||
|
f"options, so it may be incompatible: {last}")
|
||||||
|
raise YtDlpError(error or f"yt-dlp exited with status {proc.returncode}")
|
||||||
|
|
||||||
|
|
||||||
|
def bundled() -> YtDlp:
|
||||||
|
return YtDlp(importlib.metadata.version("yt-dlp"))
|
||||||
|
|
||||||
|
|
||||||
|
def _latest_release() -> str:
|
||||||
|
r = httpx.get("https://pypi.org/pypi/yt-dlp/json", timeout=15,
|
||||||
|
follow_redirects=True)
|
||||||
|
r.raise_for_status()
|
||||||
|
return r.json()["info"]["version"]
|
||||||
|
|
||||||
|
|
||||||
|
def _pip_install(target: Path, version: str) -> None:
|
||||||
|
proc = subprocess.run(
|
||||||
|
[sys.executable, "-m", "pip", "install", "--quiet", "--no-input",
|
||||||
|
"--disable-pip-version-check", "--target", str(target),
|
||||||
|
f"{REQUIREMENT}=={version}"],
|
||||||
|
capture_output=True, text=True)
|
||||||
|
if proc.returncode:
|
||||||
|
raise YtDlpError(f"pip could not install yt-dlp {version}: "
|
||||||
|
f"{proc.stderr.strip()[-500:]}")
|
||||||
|
|
||||||
|
|
||||||
|
def _install(version: str, log) -> YtDlp:
|
||||||
|
"""Installs `version` into the cache unless it is already there.
|
||||||
|
|
||||||
|
The install is checked to run before it is moved into place, so a
|
||||||
|
directory in the cache is always a usable yt-dlp."""
|
||||||
|
dest = cache_dir() / version
|
||||||
|
if dest.is_dir():
|
||||||
|
return YtDlp(version, dest)
|
||||||
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
log(f"installing yt-dlp {version}")
|
||||||
|
tmp = Path(tempfile.mkdtemp(dir=dest.parent, prefix=".tmp-"))
|
||||||
|
try:
|
||||||
|
_pip_install(tmp, version)
|
||||||
|
proc = subprocess.run(
|
||||||
|
[sys.executable, "-P", "-m", "yt_dlp", "--version"],
|
||||||
|
capture_output=True, text=True, env=YtDlp(version, tmp)._env())
|
||||||
|
got = proc.stdout.strip()
|
||||||
|
if proc.returncode or _key(got) != _key(version):
|
||||||
|
raise YtDlpError(f"yt-dlp {version} failed its check after install: "
|
||||||
|
f"{(got or proc.stderr.strip())[-500:]}")
|
||||||
|
try:
|
||||||
|
tmp.rename(dest)
|
||||||
|
except OSError:
|
||||||
|
if not dest.is_dir():
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(tmp, ignore_errors=True)
|
||||||
|
return YtDlp(version, dest)
|
||||||
|
|
||||||
|
|
||||||
|
def _cached() -> list[YtDlp]:
|
||||||
|
root = cache_dir()
|
||||||
|
if not root.is_dir():
|
||||||
|
return []
|
||||||
|
found = [YtDlp(p.name, p) for p in root.iterdir()
|
||||||
|
if p.is_dir() and re.fullmatch(r"\d+(\.\d+)*", p.name)]
|
||||||
|
return sorted(found, key=lambda y: _key(y.version))
|
||||||
|
|
||||||
|
|
||||||
|
def _newest(base: YtDlp, refresh: bool, log) -> YtDlp | None:
|
||||||
|
"""The newest cached release newer than `base`, checking PyPI for a newer
|
||||||
|
one first if that has not been done for a day or `refresh` is set."""
|
||||||
|
marker = cache_dir() / ".checked"
|
||||||
|
if (refresh or not marker.exists()
|
||||||
|
or time.time() - marker.stat().st_mtime > CHECK_INTERVAL):
|
||||||
|
# Recorded even on failure, so a broken release is not reinstalled
|
||||||
|
# for every job; a failed download refreshes regardless.
|
||||||
|
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
marker.touch()
|
||||||
|
try:
|
||||||
|
latest = _latest_release()
|
||||||
|
if _key(latest) > _key(base.version):
|
||||||
|
_install(latest, log)
|
||||||
|
except Exception as e:
|
||||||
|
log(f"could not update yt-dlp: {e}")
|
||||||
|
newer = [y for y in _cached() if _key(y.version) > _key(base.version)]
|
||||||
|
for old in newer[:-1]:
|
||||||
|
shutil.rmtree(old.path, ignore_errors=True)
|
||||||
|
return newer[-1] if newer else None
|
||||||
|
|
||||||
|
|
||||||
|
def candidates(cfg: Config, log, refresh: bool = False) -> list[YtDlp]:
|
||||||
|
"""The yt-dlp copies to try, in order: the newest release (or the one
|
||||||
|
`cfg.ytdlp_version` pins), then the bundled copy."""
|
||||||
|
base = bundled()
|
||||||
|
want = cfg.ytdlp_version
|
||||||
|
if want == "bundled":
|
||||||
|
return [base]
|
||||||
|
with _lock:
|
||||||
|
if want == "auto":
|
||||||
|
first = _newest(base, refresh, log)
|
||||||
|
elif not re.fullmatch(r"\d+(\.\d+)*", want):
|
||||||
|
raise YtDlpError(f"ytdlp_version must be \"auto\", \"bundled\" or a "
|
||||||
|
f"release such as \"2026.08.19\", not {want!r}")
|
||||||
|
elif _key(want) == _key(base.version):
|
||||||
|
first = None
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
first = _install(want, log)
|
||||||
|
except Exception as e:
|
||||||
|
log(f"could not install the pinned yt-dlp {want}: {e}")
|
||||||
|
first = None
|
||||||
|
return [first, base] if first else [base]
|
||||||
+79
-54
@@ -91,87 +91,112 @@ def test_classify_ignores_partials_left_by_a_failed_attempt(tmp_path):
|
|||||||
video, _ = acquire._classify_downloads(tmp_path)
|
video, _ = acquire._classify_downloads(tmp_path)
|
||||||
assert video.name == "source_video.webm"
|
assert video.name == "source_video.webm"
|
||||||
|
|
||||||
def test_ydl_opts_default_client_adds_no_player_client(tmp_path):
|
def test_ydl_args_default_client_adds_no_player_client(tmp_path):
|
||||||
cfg = Config(whisper_url="http://x/v1")
|
cfg = Config(whisper_url="http://x/v1")
|
||||||
opts = acquire._ydl_opts(cfg, tmp_path, "default", print)
|
args = acquire._ydl_args(cfg, tmp_path, "default", "https://yt/x")
|
||||||
assert "youtube" not in opts["extractor_args"]
|
assert "--extractor-args" not in args
|
||||||
assert opts["retries"] and opts["fragment_retries"] and opts["extractor_retries"]
|
assert "--retries" in args and "--fragment-retries" in args
|
||||||
|
assert args[-2:] == ["--", "https://yt/x"]
|
||||||
|
|
||||||
def test_ydl_opts_named_client_and_pot_provider(tmp_path):
|
def test_ydl_args_named_client_pot_provider_and_cookies(tmp_path):
|
||||||
cfg = Config(whisper_url="http://x/v1",
|
cfg = Config(whisper_url="http://x/v1",
|
||||||
ytdlp_pot_provider_url="http://pot:4416",
|
ytdlp_pot_provider_url="http://pot:4416",
|
||||||
ytdlp_cookies=tmp_path / "cookies.txt")
|
ytdlp_cookies=tmp_path / "cookies.txt",
|
||||||
opts = acquire._ydl_opts(cfg, tmp_path, "tv", print)
|
ytdlp_impersonate="chrome")
|
||||||
assert opts["extractor_args"]["youtube"] == {"player_client": ["tv"]}
|
args = acquire._ydl_args(cfg, tmp_path, "tv", "https://yt/x")
|
||||||
assert opts["extractor_args"]["youtubepot-bgutilhttp"] == {
|
joined = " ".join(args)
|
||||||
"base_url": ["http://pot:4416"]}
|
assert "--extractor-args youtube:player_client=tv" in joined
|
||||||
assert opts["cookiefile"] == str(tmp_path / "cookies.txt")
|
assert ("--extractor-args youtubepot-bgutilhttp:base_url=http://pot:4416"
|
||||||
|
in joined)
|
||||||
|
assert f"--cookies {tmp_path / 'cookies.txt'}" in joined
|
||||||
|
assert "--impersonate chrome" in joined
|
||||||
|
|
||||||
def _fake_yt_dlp(monkeypatch, behaviour):
|
def test_ydl_args_parse_with_real_yt_dlp(tmp_path):
|
||||||
"""Install a stub yt_dlp whose download calls `behaviour(opts)`."""
|
"""Every option songtrace passes must exist in the bundled yt-dlp."""
|
||||||
import sys
|
import pytest
|
||||||
import types
|
pytest.importorskip("yt_dlp")
|
||||||
|
from yt_dlp import parse_options
|
||||||
|
cfg = Config(whisper_url="http://x/v1", ytdlp_pot_provider_url="http://p:1",
|
||||||
|
ytdlp_cookies=tmp_path / "c.txt")
|
||||||
|
parse_options(acquire._ydl_args(cfg, tmp_path, "tv", "https://yt/x"))
|
||||||
|
|
||||||
|
class FakeYtDlp:
|
||||||
|
def __init__(self, version, behaviour, seen):
|
||||||
|
self.version, self._behaviour, self._seen = version, behaviour, seen
|
||||||
|
def run(self, args, log):
|
||||||
|
client = next((a for a in args if a.startswith("youtube:")), None)
|
||||||
|
self._seen.append((self.version, client))
|
||||||
|
self._behaviour(self.version, client)
|
||||||
|
|
||||||
|
def _fake_candidates(monkeypatch, behaviour, versions=("2026.09.01", "2026.08.19"),
|
||||||
|
refreshed=None):
|
||||||
|
"""Stub ytdlp.candidates; `refreshed` is what a refresh turns up."""
|
||||||
seen = []
|
seen = []
|
||||||
|
def candidates(cfg, log, refresh=False):
|
||||||
class FakeYoutubeDL:
|
vs = refreshed if refresh and refreshed else versions
|
||||||
def __init__(self, opts):
|
return [FakeYtDlp(v, behaviour, seen) for v in vs]
|
||||||
self.opts = opts
|
monkeypatch.setattr(acquire.ytdlp, "candidates", candidates)
|
||||||
def __enter__(self):
|
|
||||||
return self
|
|
||||||
def __exit__(self, *exc):
|
|
||||||
return False
|
|
||||||
def extract_info(self, url, download):
|
|
||||||
seen.append(self.opts["extractor_args"].get("youtube"))
|
|
||||||
behaviour(self.opts)
|
|
||||||
|
|
||||||
mod = types.ModuleType("yt_dlp")
|
|
||||||
mod.YoutubeDL = FakeYoutubeDL
|
|
||||||
mod.version = types.SimpleNamespace(__version__="2026.08.19")
|
|
||||||
monkeypatch.setitem(sys.modules, "yt_dlp", mod)
|
|
||||||
return seen
|
return seen
|
||||||
|
|
||||||
def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch):
|
def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch):
|
||||||
def behaviour(opts):
|
from songtrace.ytdlp import YtDlpError
|
||||||
client = opts["extractor_args"].get("youtube")
|
def behaviour(version, client):
|
||||||
if client is None:
|
if client is None:
|
||||||
raise RuntimeError("HTTP Error 403: Forbidden")
|
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
|
||||||
(tmp_path / "source_video.webm").write_bytes(b"v")
|
(tmp_path / "source_video.webm").write_bytes(b"v")
|
||||||
seen = _fake_yt_dlp(monkeypatch, behaviour)
|
seen = _fake_candidates(monkeypatch, behaviour)
|
||||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
|
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
|
||||||
logged = []
|
logged = []
|
||||||
video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg,
|
video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg,
|
||||||
logged.append)
|
logged.append)
|
||||||
assert video.name == "source_video.webm"
|
assert video.name == "source_video.webm"
|
||||||
assert thumb is None
|
assert thumb is None
|
||||||
assert seen == [None, {"player_client": ["tv"]}]
|
assert seen == [("2026.09.01", None),
|
||||||
|
("2026.09.01", "youtube:player_client=tv")]
|
||||||
assert any("403" in m for m in logged)
|
assert any("403" in m for m in logged)
|
||||||
|
|
||||||
def test_download_reports_version_and_clients_when_all_fail(tmp_path, monkeypatch):
|
def test_download_falls_back_to_the_bundled_copy(tmp_path, monkeypatch):
|
||||||
|
from songtrace.ytdlp import YtDlpError
|
||||||
|
def behaviour(version, client):
|
||||||
|
if version == "2026.09.01":
|
||||||
|
raise YtDlpError("rejected songtrace's options")
|
||||||
|
(tmp_path / "source_video.webm").write_bytes(b"v")
|
||||||
|
seen = _fake_candidates(monkeypatch, behaviour)
|
||||||
|
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default"])
|
||||||
|
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||||
|
assert [v for v, _ in seen] == ["2026.09.01", "2026.08.19"]
|
||||||
|
|
||||||
|
def test_download_retries_with_a_release_found_after_failing(tmp_path, monkeypatch):
|
||||||
|
from songtrace.ytdlp import YtDlpError
|
||||||
|
def behaviour(version, client):
|
||||||
|
if version != "2026.10.01":
|
||||||
|
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
|
||||||
|
(tmp_path / "source_video.webm").write_bytes(b"v")
|
||||||
|
seen = _fake_candidates(monkeypatch, behaviour,
|
||||||
|
refreshed=("2026.10.01", "2026.08.19"))
|
||||||
|
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default"])
|
||||||
|
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||||
|
assert [v for v, _ in seen] == ["2026.09.01", "2026.08.19", "2026.10.01"]
|
||||||
|
|
||||||
|
def test_download_reports_versions_and_clients_when_all_fail(tmp_path, monkeypatch):
|
||||||
import pytest
|
import pytest
|
||||||
from songtrace.pipeline import PipelineError
|
from songtrace.pipeline import PipelineError
|
||||||
def behaviour(opts):
|
from songtrace.ytdlp import YtDlpError
|
||||||
raise RuntimeError("HTTP Error 403: Forbidden")
|
def behaviour(version, client):
|
||||||
_fake_yt_dlp(monkeypatch, behaviour)
|
raise YtDlpError("ERROR: HTTP Error 403: Forbidden")
|
||||||
|
seen = _fake_candidates(monkeypatch, behaviour)
|
||||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
|
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
|
||||||
with pytest.raises(PipelineError) as e:
|
with pytest.raises(PipelineError) as e:
|
||||||
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||||
assert "2026.08.19" in str(e.value)
|
assert "2026.09.01, 2026.08.19" in str(e.value)
|
||||||
assert "default, tv" in str(e.value)
|
assert "default, tv" in str(e.value)
|
||||||
assert "403" in str(e.value)
|
assert "403" in str(e.value)
|
||||||
|
assert len(seen) == 4
|
||||||
|
|
||||||
def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch):
|
def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch):
|
||||||
seen = _fake_yt_dlp(
|
seen = _fake_candidates(
|
||||||
monkeypatch, lambda opts: (tmp_path / "source_video.mp4").write_bytes(b"v"))
|
monkeypatch,
|
||||||
|
lambda v, c: (tmp_path / "source_video.mp4").write_bytes(b"v"))
|
||||||
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[])
|
cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[])
|
||||||
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
|
||||||
assert seen == [None]
|
assert seen == [("2026.09.01", None)]
|
||||||
|
|
||||||
def test_ydl_log_drops_debug_lines():
|
|
||||||
logged = []
|
|
||||||
log = acquire._YdlLog(logged.append)
|
|
||||||
log.debug("[debug] internal detail")
|
|
||||||
log.debug("[youtube] Extracting URL")
|
|
||||||
log.warning("nsig extraction failed")
|
|
||||||
log.error("HTTP Error 403: Forbidden")
|
|
||||||
assert logged == ["[youtube] Extracting URL", "nsig extraction failed",
|
|
||||||
"HTTP Error 403: Forbidden"]
|
|
||||||
|
|||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import os
|
||||||
|
import time
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from songtrace import ytdlp
|
||||||
|
from songtrace.config import Config
|
||||||
|
from songtrace.ytdlp import YtDlp, YtDlpError
|
||||||
|
|
||||||
|
|
||||||
|
def fake_package(path, version="2026.09.01", body=""):
|
||||||
|
"""A stand-in yt_dlp that answers --version and otherwise runs `body`."""
|
||||||
|
pkg = path / "yt_dlp"
|
||||||
|
pkg.mkdir(parents=True)
|
||||||
|
(pkg / "__init__.py").touch()
|
||||||
|
(pkg / "__main__.py").write_text(
|
||||||
|
"import sys\n"
|
||||||
|
f"if '--version' in sys.argv: print({version!r}); sys.exit()\n"
|
||||||
|
+ body)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def cache(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache"))
|
||||||
|
monkeypatch.setattr(ytdlp, "bundled", lambda: YtDlp("2026.8.19"))
|
||||||
|
return ytdlp.cache_dir()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def pypi(monkeypatch):
|
||||||
|
"""Stubs PyPI and pip: `latest` is the newest release, `installs` logs pip."""
|
||||||
|
state = {"latest": "2026.9.1", "installs": []}
|
||||||
|
monkeypatch.setattr(ytdlp, "_latest_release", lambda: state["latest"])
|
||||||
|
def pip_install(target, version):
|
||||||
|
state["installs"].append(version)
|
||||||
|
fake_package(target, version)
|
||||||
|
monkeypatch.setattr(ytdlp, "_pip_install", pip_install)
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def cfg(version="auto"):
|
||||||
|
return Config(whisper_url="http://x/v1", ytdlp_version=version)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_uses_the_install_on_its_path_and_logs_its_output(tmp_path):
|
||||||
|
path = fake_package(tmp_path, body="print('[youtube] hello', sys.argv[1:])\n")
|
||||||
|
logged = []
|
||||||
|
YtDlp("2026.09.01", path).run(["--", "https://yt/x"], logged.append)
|
||||||
|
assert logged == ["[youtube] hello ['--ignore-config', '--', 'https://yt/x']"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_raises_the_error_line(tmp_path):
|
||||||
|
path = fake_package(tmp_path, body=(
|
||||||
|
"print('ERROR: HTTP Error 403: Forbidden', file=sys.stderr)\n"
|
||||||
|
"print('trailing noise')\nsys.exit(1)\n"))
|
||||||
|
with pytest.raises(YtDlpError, match="^ERROR: HTTP Error 403"):
|
||||||
|
YtDlp("2026.09.01", path).run([], lambda m: None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_flags_a_usage_error_as_incompatible(tmp_path):
|
||||||
|
path = fake_package(tmp_path, body=(
|
||||||
|
"print('error: no such option: --retries', file=sys.stderr)\n"
|
||||||
|
"sys.exit(2)\n"))
|
||||||
|
with pytest.raises(YtDlpError, match="2026.09.01 rejected.*--retries"):
|
||||||
|
YtDlp("2026.09.01", path).run([], lambda m: None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_installs_a_newer_release_and_tries_it_first(cache, pypi):
|
||||||
|
found = ytdlp.candidates(cfg(), lambda m: None)
|
||||||
|
assert [y.version for y in found] == ["2026.9.1", "2026.8.19"]
|
||||||
|
assert found[0].path == cache / "2026.9.1"
|
||||||
|
assert not [p for p in cache.iterdir() if p.name.startswith(".tmp-")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_checks_pypi_once_a_day(cache, pypi):
|
||||||
|
ytdlp.candidates(cfg(), lambda m: None)
|
||||||
|
pypi["latest"] = "2026.10.1"
|
||||||
|
found = ytdlp.candidates(cfg(), lambda m: None)
|
||||||
|
assert found[0].version == "2026.9.1"
|
||||||
|
found = ytdlp.candidates(cfg(), lambda m: None, refresh=True)
|
||||||
|
assert found[0].version == "2026.10.1"
|
||||||
|
assert not (cache / "2026.9.1").exists()
|
||||||
|
old = time.time() - ytdlp.CHECK_INTERVAL - 1
|
||||||
|
os.utime(cache / ".checked", (old, old))
|
||||||
|
pypi["latest"] = "2026.11.1"
|
||||||
|
assert ytdlp.candidates(cfg(), lambda m: None)[0].version == "2026.11.1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_uses_only_bundled_when_it_is_newest(cache, pypi):
|
||||||
|
pypi["latest"] = "2026.8.19"
|
||||||
|
assert [y.version for y in ytdlp.candidates(cfg(), lambda m: None)] == [
|
||||||
|
"2026.8.19"]
|
||||||
|
assert pypi["installs"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_auto_survives_pypi_being_unreachable(cache, pypi, monkeypatch):
|
||||||
|
ytdlp.candidates(cfg(), lambda m: None)
|
||||||
|
def down():
|
||||||
|
raise OSError("network unreachable")
|
||||||
|
monkeypatch.setattr(ytdlp, "_latest_release", down)
|
||||||
|
logged = []
|
||||||
|
found = ytdlp.candidates(cfg(), logged.append, refresh=True)
|
||||||
|
assert [y.version for y in found] == ["2026.9.1", "2026.8.19"]
|
||||||
|
assert any("network unreachable" in m for m in logged)
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_install_that_fails_its_check_is_discarded(cache, pypi, monkeypatch):
|
||||||
|
monkeypatch.setattr(ytdlp, "_pip_install",
|
||||||
|
lambda target, v: fake_package(target, "2026.1.1"))
|
||||||
|
logged = []
|
||||||
|
assert [y.version for y in ytdlp.candidates(cfg(), logged.append)] == [
|
||||||
|
"2026.8.19"]
|
||||||
|
assert any("failed its check" in m for m in logged)
|
||||||
|
assert list(cache.iterdir()) == [cache / ".checked"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_bundled_never_touches_pypi(cache, pypi):
|
||||||
|
pypi["latest"] = None
|
||||||
|
assert [y.version for y in ytdlp.candidates(cfg("bundled"), print)] == [
|
||||||
|
"2026.8.19"]
|
||||||
|
assert not cache.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_installs_that_release_even_if_older(cache, pypi):
|
||||||
|
found = ytdlp.candidates(cfg("2026.07.01"), lambda m: None)
|
||||||
|
assert [y.version for y in found] == ["2026.07.01", "2026.8.19"]
|
||||||
|
assert pypi["installs"] == ["2026.07.01"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_matching_bundled_installs_nothing(cache, pypi):
|
||||||
|
found = ytdlp.candidates(cfg("2026.08.19"), lambda m: None)
|
||||||
|
assert [y.version for y in found] == ["2026.8.19"]
|
||||||
|
assert pypi["installs"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_pin_must_be_a_release_number(cache, pypi):
|
||||||
|
with pytest.raises(YtDlpError, match="ytdlp_version"):
|
||||||
|
ytdlp.candidates(cfg("latest; rm -rf /"), lambda m: None)
|
||||||
Reference in New Issue
Block a user