Merge pull request 'Fix YouTube 403s: install yt-dlp's YouTube extras and retry across player clients' (#8) from ytdlp-resilience into main

Reviewed-on: #8
Reviewed-by: branderson <brad@branderson.io>

README.md

@@ -51,7 +51,9 @@ $EDITOR config.toml   # set whisper_url at minimum
 
 See `config.example.toml` for every available field (`whisper_url`,
 `whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`,
-`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`).
+`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`,
+`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`,
+`ytdlp_pot_provider_url`).
 
 `whisper_api` selects the wire format: `"openai"` (default) speaks the
 OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"`
@@ -117,6 +119,46 @@ songtrace make --title "..." --artist "..." --lyrics lyrics.txt \
 Valid stages, in pipeline order: `acquire`, `separate`, `transcribe`,
 `align`, `pitch`, `notes`, `grid`, `package`.
 
+## YouTube download failures
+
+`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
+page but refused the media URL. In rough order of likelihood:
+
+1. **yt-dlp is stale.** YouTube changes its defences every few weeks and the
+   version baked into the image does not. Hotfix a running container, then
+   rebuild so it sticks:
+
+   ```sh
+   docker compose exec songtrace pip install -U 'yt-dlp[default,curl-cffi,deno]'
+   docker compose restart songtrace
+   docker compose build --pull songtrace   # permanent
+   ```
+
+2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine
+   to solve YouTube's player challenges; without them it silently falls back to
+   formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml`
+   cover this. Check with
+   `docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
+   header must show a `JS runtimes:` line that is not `none`.
+
+3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get
+   throttled hardest. Try from another network before changing config.
+
+4. **A PO token is required.** If the log says so explicitly, run
+   [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider)
+   as a sidecar, install its plugin into the image, and point
+   `ytdlp_pot_provider_url` at it.
+
+`ytdlp_clients` retries the download against each listed player client in
+turn, so a client that YouTube has broken is skipped automatically.
+`ytdlp_cookies` (a Netscape cookie jar, for age-gated videos) and
+`ytdlp_impersonate` (a browser TLS fingerprint, e.g. `"chrome"`) are the other
+two levers. Do not use a cookie jar from an account you care about — YouTube
+bans for automated access.
+
+Failed downloads leave the work dir intact, so re-running the job resumes
+rather than starting over.
+
 ## Regenerating the web UI screenshot
 
 `scripts/screenshot_webui.py` seeds the real app with a fixed set of jobs

config.example.toml

@@ -11,3 +11,10 @@ creator = "brad"
 # device = "cuda"
 bind_host = "0.0.0.0"
 # port = 8000
+
+# YouTube downloads. Each player client is tried in order until one works;
+# "default" means whatever yt-dlp currently picks for itself.
+# ytdlp_clients = ["default", "tv"]
+# ytdlp_cookies = "/config/youtube-cookies.txt"  # Netscape cookie jar
+# ytdlp_impersonate = "chrome"                   # TLS fingerprint to present
+# ytdlp_pot_provider_url = "http://bgutil-provider:4416"

pyproject.toml

@@ -5,7 +5,12 @@ description = "Generate UltraStar/Melody Mania karaoke songs from a video and re
 requires-python = ">=3.11"
 license = "AGPL-3.0-or-later"
 dependencies = [
-    "yt-dlp",
+    # yt-dlp declares no runtime deps of its own; the extras are what make
+    # YouTube work. "default" pulls yt-dlp-ejs (JS challenge solving) and the
+    # requests/urllib3 handlers, "deno" ships the JS runtime that executes it,
+    # and "curl-cffi" provides the --impersonate targets. Without them YouTube
+    # extraction degrades to HTTP 403 on the media URLs.
+    "yt-dlp[default,curl-cffi,deno]",
     "demucs",
     "torch",
     "torchaudio",

src/songtrace/config.py

@@ -1,6 +1,6 @@
 import os
 import tomllib
-from dataclasses import dataclass, fields
+from dataclasses import dataclass, field, fields
 from pathlib import Path
 
 
@@ -18,9 +18,15 @@ class Config:
     device: str = "cuda"
     bind_host: str = "127.0.0.1"
     port: int = 8000
+    # YouTube player clients to try, in order, until one yields a download.
+    # "default" means yt-dlp's own choice; see its extractor-args docs.
+    ytdlp_clients: list[str] = field(default_factory=lambda: ["default", "tv"])
+    ytdlp_cookies: Path | None = None
+    ytdlp_impersonate: str | None = None
+    ytdlp_pot_provider_url: str | None = None
 
 
-_PATH_FIELDS = {"output_dir", "work_dir"}
+_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"}
 
 
 def load_config(path: Path | None = None) -> Config:

src/songtrace/stages/acquire.py

@@ -2,14 +2,18 @@ import json
 import shutil
 from pathlib import Path
 
+from ..config import Config
 from ..media import ffmpeg
 from ..pipeline import Ctx, PipelineError, Stage
 
 IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"}
+# yt-dlp's in-progress scratch files, which a failed attempt leaves behind.
+PARTIAL_EXTENSIONS = {".part", ".ytdl", ".temp"}
 
 
 def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
-    files = list(dest.glob("source_video.*"))
+    files = [p for p in dest.glob("source_video.*")
+             if p.suffix.lower() not in PARTIAL_EXTENSIONS]
     thumbs = [p for p in files if p.suffix.lower() in IMAGE_EXTENSIONS]
     videos = [p for p in files if p.suffix.lower() not in IMAGE_EXTENSIONS]
     if not videos:
@@ -18,17 +22,67 @@ def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
     return video, thumbs[0] if thumbs else None
 
 
-def download_youtube(url: str, dest: Path) -> tuple[Path, Path | None]:
-    import yt_dlp
+class _YdlLog:
+    """Routes yt-dlp's own messages into the job log, minus progress noise."""
+
+    def __init__(self, log):
+        self._log = log
+
+    def debug(self, msg: str) -> None:
+        if not msg.startswith("[debug] "):
+            self._log(msg)
+
+    def warning(self, msg: str) -> None:
+        self._log(msg)
+
+    error = warning
+
+
+def _ydl_opts(cfg: Config, dest: Path, client: str, log) -> dict:
+    extractor_args: dict[str, dict[str, list[str]]] = {}
+    if client != "default":
+        extractor_args["youtube"] = {"player_client": [client]}
+    if cfg.ytdlp_pot_provider_url:
+        extractor_args["youtubepot-bgutilhttp"] = {
+            "base_url": [cfg.ytdlp_pot_provider_url]}
     opts = {
         "outtmpl": str(dest / "source_video.%(ext)s"),
         "format": "bestvideo*+bestaudio/best",
         "writethumbnail": True,
         "noplaylist": True,
+        "noprogress": True,
+        "logger": _YdlLog(log),
+        "retries": 10,
+        "fragment_retries": 10,
+        "extractor_retries": 5,
+        "extractor_args": extractor_args,
     }
-    with yt_dlp.YoutubeDL(opts) as ydl:
-        ydl.extract_info(url, download=True)
-    return _classify_downloads(dest)
+    if cfg.ytdlp_cookies:
+        opts["cookiefile"] = str(cfg.ytdlp_cookies)
+    if cfg.ytdlp_impersonate:
+        from yt_dlp.networking.impersonate import ImpersonateTarget
+        opts["impersonate"] = ImpersonateTarget.from_str(cfg.ytdlp_impersonate)
+    return opts
+
+
+def download_youtube(url: str, dest: Path, cfg: Config,
+                     log=print) -> tuple[Path, Path | None]:
+    import yt_dlp
+    clients = list(cfg.ytdlp_clients) or ["default"]
+    last: Exception | None = None
+    for client in clients:
+        try:
+            with yt_dlp.YoutubeDL(_ydl_opts(cfg, dest, client, log)) as ydl:
+                ydl.extract_info(url, download=True)
+            return _classify_downloads(dest)
+        except Exception as e:
+            last = e
+            log(f"yt-dlp player_client={client} failed: {e}")
+    raise PipelineError(
+        f"yt-dlp {yt_dlp.version.__version__} could not download {url} after "
+        f"trying player_client={', '.join(clients)}. Last error: {last}. "
+        "YouTube blocks change often: upgrade yt-dlp, then see the yt-dlp "
+        "options in config.toml (cookies, impersonation, PO token provider).")
 
 
 def _run(ctx: Ctx) -> None:
@@ -36,7 +90,8 @@ def _run(ctx: Ctx) -> None:
         raise PipelineError("provide exactly one of youtube_url / video_path")
     thumb = None
     if ctx.inp.youtube_url:
-        video, thumb = download_youtube(ctx.inp.youtube_url, ctx.work)
+        video, thumb = download_youtube(ctx.inp.youtube_url, ctx.work,
+                                       ctx.cfg, ctx.log)
     else:
         video = ctx.work / f"source_video{ctx.inp.video_path.suffix}"
         shutil.copyfile(ctx.inp.video_path, video)

tests/test_acquire.py

@@ -24,7 +24,7 @@ def test_local_video(tmp_path, monkeypatch):
     assert any("audio.wav" in str(a[-1]) for a in calls)
 
 def test_youtube(tmp_path, monkeypatch):
-    def fake_dl(url, dest):
+    def fake_dl(url, dest, cfg, log):
         video = dest / "source_video.webm"
         video.write_bytes(b"v")
         thumb = dest / "source_video.jpg"
@@ -83,3 +83,95 @@ def test_classify_picks_largest_video_when_multiple(tmp_path):
     (tmp_path / "source_video.mp4").write_bytes(b"much larger contents here")
     video, _ = acquire._classify_downloads(tmp_path)
     assert video.name == "source_video.mp4"
+
+def test_classify_ignores_partials_left_by_a_failed_attempt(tmp_path):
+    (tmp_path / "source_video.webm").write_bytes(b"v")
+    (tmp_path / "source_video.f137.mp4.part").write_bytes(b"a much bigger partial")
+    (tmp_path / "source_video.f251.webm.ytdl").write_bytes(b"resume state")
+    video, _ = acquire._classify_downloads(tmp_path)
+    assert video.name == "source_video.webm"
+
+def test_ydl_opts_default_client_adds_no_player_client(tmp_path):
+    cfg = Config(whisper_url="http://x/v1")
+    opts = acquire._ydl_opts(cfg, tmp_path, "default", print)
+    assert "youtube" not in opts["extractor_args"]
+    assert opts["retries"] and opts["fragment_retries"] and opts["extractor_retries"]
+
+def test_ydl_opts_named_client_and_pot_provider(tmp_path):
+    cfg = Config(whisper_url="http://x/v1",
+                 ytdlp_pot_provider_url="http://pot:4416",
+                 ytdlp_cookies=tmp_path / "cookies.txt")
+    opts = acquire._ydl_opts(cfg, tmp_path, "tv", print)
+    assert opts["extractor_args"]["youtube"] == {"player_client": ["tv"]}
+    assert opts["extractor_args"]["youtubepot-bgutilhttp"] == {
+        "base_url": ["http://pot:4416"]}
+    assert opts["cookiefile"] == str(tmp_path / "cookies.txt")
+
+def _fake_yt_dlp(monkeypatch, behaviour):
+    """Install a stub yt_dlp whose download calls `behaviour(opts)`."""
+    import sys
+    import types
+    seen = []
+
+    class FakeYoutubeDL:
+        def __init__(self, opts):
+            self.opts = opts
+        def __enter__(self):
+            return self
+        def __exit__(self, *exc):
+            return False
+        def extract_info(self, url, download):
+            seen.append(self.opts["extractor_args"].get("youtube"))
+            behaviour(self.opts)
+
+    mod = types.ModuleType("yt_dlp")
+    mod.YoutubeDL = FakeYoutubeDL
+    mod.version = types.SimpleNamespace(__version__="2026.08.19")
+    monkeypatch.setitem(sys.modules, "yt_dlp", mod)
+    return seen
+
+def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch):
+    def behaviour(opts):
+        client = opts["extractor_args"].get("youtube")
+        if client is None:
+            raise RuntimeError("HTTP Error 403: Forbidden")
+        (tmp_path / "source_video.webm").write_bytes(b"v")
+    seen = _fake_yt_dlp(monkeypatch, behaviour)
+    cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
+    logged = []
+    video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg,
+                                            logged.append)
+    assert video.name == "source_video.webm"
+    assert thumb is None
+    assert seen == [None, {"player_client": ["tv"]}]
+    assert any("403" in m for m in logged)
+
+def test_download_reports_version_and_clients_when_all_fail(tmp_path, monkeypatch):
+    import pytest
+    from songtrace.pipeline import PipelineError
+    def behaviour(opts):
+        raise RuntimeError("HTTP Error 403: Forbidden")
+    _fake_yt_dlp(monkeypatch, behaviour)
+    cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
+    with pytest.raises(PipelineError) as e:
+        acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
+    assert "2026.08.19" in str(e.value)
+    assert "default, tv" in str(e.value)
+    assert "403" in str(e.value)
+
+def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch):
+    seen = _fake_yt_dlp(
+        monkeypatch, lambda opts: (tmp_path / "source_video.mp4").write_bytes(b"v"))
+    cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[])
+    acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
+    assert seen == [None]
+
+def test_ydl_log_drops_debug_lines():
+    logged = []
+    log = acquire._YdlLog(logged.append)
+    log.debug("[debug] internal detail")
+    log.debug("[youtube] Extracting URL")
+    log.warning("nsig extraction failed")
+    log.error("HTTP Error 403: Forbidden")
+    assert logged == ["[youtube] Extracting URL", "nsig extraction failed",
+                      "HTTP Error 403: Forbidden"]