diff --git a/README.md b/README.md index e6e86b2..4c5ab5a 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,9 @@ $EDITOR config.toml # set whisper_url at minimum See `config.example.toml` for every available field (`whisper_url`, `whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`, -`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`). +`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`, +`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`, +`ytdlp_pot_provider_url`). `whisper_api` selects the wire format: `"openai"` (default) speaks the OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"` @@ -117,6 +119,46 @@ songtrace make --title "..." --artist "..." --lyrics lyrics.txt \ Valid stages, in pipeline order: `acquire`, `separate`, `transcribe`, `align`, `pitch`, `notes`, `grid`, `package`. +## YouTube download failures + +`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the +page but refused the media URL. In rough order of likelihood: + +1. **yt-dlp is stale.** YouTube changes its defences every few weeks and the + version baked into the image does not. Hotfix a running container, then + rebuild so it sticks: + + ```sh + docker compose exec songtrace pip install -U 'yt-dlp[default,curl-cffi,deno]' + docker compose restart songtrace + docker compose build --pull songtrace # permanent + ``` + +2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine + to solve YouTube's player challenges; without them it silently falls back to + formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml` + cover this. Check with + `docker compose exec songtrace yt-dlp -v --simulate ` — the debug + header must show a `JS runtimes:` line that is not `none`. + +3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get + throttled hardest. Try from another network before changing config. + +4. **A PO token is required.** If the log says so explicitly, run + [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider) + as a sidecar, install its plugin into the image, and point + `ytdlp_pot_provider_url` at it. + +`ytdlp_clients` retries the download against each listed player client in +turn, so a client that YouTube has broken is skipped automatically. +`ytdlp_cookies` (a Netscape cookie jar, for age-gated videos) and +`ytdlp_impersonate` (a browser TLS fingerprint, e.g. `"chrome"`) are the other +two levers. Do not use a cookie jar from an account you care about — YouTube +bans for automated access. + +Failed downloads leave the work dir intact, so re-running the job resumes +rather than starting over. + ## Regenerating the web UI screenshot `scripts/screenshot_webui.py` seeds the real app with a fixed set of jobs diff --git a/config.example.toml b/config.example.toml index 8ade36c..0d61f49 100644 --- a/config.example.toml +++ b/config.example.toml @@ -11,3 +11,10 @@ creator = "brad" # device = "cuda" bind_host = "0.0.0.0" # port = 8000 + +# YouTube downloads. Each player client is tried in order until one works; +# "default" means whatever yt-dlp currently picks for itself. +# ytdlp_clients = ["default", "tv"] +# ytdlp_cookies = "/config/youtube-cookies.txt" # Netscape cookie jar +# ytdlp_impersonate = "chrome" # TLS fingerprint to present +# ytdlp_pot_provider_url = "http://bgutil-provider:4416" diff --git a/pyproject.toml b/pyproject.toml index 7beb5b7..deb90c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,12 @@ description = "Generate UltraStar/Melody Mania karaoke songs from a video and re requires-python = ">=3.11" license = "AGPL-3.0-or-later" dependencies = [ - "yt-dlp", + # yt-dlp declares no runtime deps of its own; the extras are what make + # YouTube work. "default" pulls yt-dlp-ejs (JS challenge solving) and the + # requests/urllib3 handlers, "deno" ships the JS runtime that executes it, + # and "curl-cffi" provides the --impersonate targets. Without them YouTube + # extraction degrades to HTTP 403 on the media URLs. + "yt-dlp[default,curl-cffi,deno]", "demucs", "torch", "torchaudio", diff --git a/src/songtrace/config.py b/src/songtrace/config.py index 5bea78d..b8e5099 100644 --- a/src/songtrace/config.py +++ b/src/songtrace/config.py @@ -1,6 +1,6 @@ import os import tomllib -from dataclasses import dataclass, fields +from dataclasses import dataclass, field, fields from pathlib import Path @@ -18,9 +18,15 @@ class Config: device: str = "cuda" bind_host: str = "127.0.0.1" port: int = 8000 + # YouTube player clients to try, in order, until one yields a download. + # "default" means yt-dlp's own choice; see its extractor-args docs. + ytdlp_clients: list[str] = field(default_factory=lambda: ["default", "tv"]) + ytdlp_cookies: Path | None = None + ytdlp_impersonate: str | None = None + ytdlp_pot_provider_url: str | None = None -_PATH_FIELDS = {"output_dir", "work_dir"} +_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"} def load_config(path: Path | None = None) -> Config: diff --git a/src/songtrace/stages/acquire.py b/src/songtrace/stages/acquire.py index 418bfa8..8733458 100644 --- a/src/songtrace/stages/acquire.py +++ b/src/songtrace/stages/acquire.py @@ -2,14 +2,18 @@ import json import shutil from pathlib import Path +from ..config import Config from ..media import ffmpeg from ..pipeline import Ctx, PipelineError, Stage IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"} +# yt-dlp's in-progress scratch files, which a failed attempt leaves behind. +PARTIAL_EXTENSIONS = {".part", ".ytdl", ".temp"} def _classify_downloads(dest: Path) -> tuple[Path, Path | None]: - files = list(dest.glob("source_video.*")) + files = [p for p in dest.glob("source_video.*") + if p.suffix.lower() not in PARTIAL_EXTENSIONS] thumbs = [p for p in files if p.suffix.lower() in IMAGE_EXTENSIONS] videos = [p for p in files if p.suffix.lower() not in IMAGE_EXTENSIONS] if not videos: @@ -18,17 +22,67 @@ def _classify_downloads(dest: Path) -> tuple[Path, Path | None]: return video, thumbs[0] if thumbs else None -def download_youtube(url: str, dest: Path) -> tuple[Path, Path | None]: - import yt_dlp +class _YdlLog: + """Routes yt-dlp's own messages into the job log, minus progress noise.""" + + def __init__(self, log): + self._log = log + + def debug(self, msg: str) -> None: + if not msg.startswith("[debug] "): + self._log(msg) + + def warning(self, msg: str) -> None: + self._log(msg) + + error = warning + + +def _ydl_opts(cfg: Config, dest: Path, client: str, log) -> dict: + extractor_args: dict[str, dict[str, list[str]]] = {} + if client != "default": + extractor_args["youtube"] = {"player_client": [client]} + if cfg.ytdlp_pot_provider_url: + extractor_args["youtubepot-bgutilhttp"] = { + "base_url": [cfg.ytdlp_pot_provider_url]} opts = { "outtmpl": str(dest / "source_video.%(ext)s"), "format": "bestvideo*+bestaudio/best", "writethumbnail": True, "noplaylist": True, + "noprogress": True, + "logger": _YdlLog(log), + "retries": 10, + "fragment_retries": 10, + "extractor_retries": 5, + "extractor_args": extractor_args, } - with yt_dlp.YoutubeDL(opts) as ydl: - ydl.extract_info(url, download=True) - return _classify_downloads(dest) + if cfg.ytdlp_cookies: + opts["cookiefile"] = str(cfg.ytdlp_cookies) + if cfg.ytdlp_impersonate: + from yt_dlp.networking.impersonate import ImpersonateTarget + opts["impersonate"] = ImpersonateTarget.from_str(cfg.ytdlp_impersonate) + return opts + + +def download_youtube(url: str, dest: Path, cfg: Config, + log=print) -> tuple[Path, Path | None]: + import yt_dlp + clients = list(cfg.ytdlp_clients) or ["default"] + last: Exception | None = None + for client in clients: + try: + with yt_dlp.YoutubeDL(_ydl_opts(cfg, dest, client, log)) as ydl: + ydl.extract_info(url, download=True) + return _classify_downloads(dest) + except Exception as e: + last = e + log(f"yt-dlp player_client={client} failed: {e}") + raise PipelineError( + f"yt-dlp {yt_dlp.version.__version__} could not download {url} after " + f"trying player_client={', '.join(clients)}. Last error: {last}. " + "YouTube blocks change often: upgrade yt-dlp, then see the yt-dlp " + "options in config.toml (cookies, impersonation, PO token provider).") def _run(ctx: Ctx) -> None: @@ -36,7 +90,8 @@ def _run(ctx: Ctx) -> None: raise PipelineError("provide exactly one of youtube_url / video_path") thumb = None if ctx.inp.youtube_url: - video, thumb = download_youtube(ctx.inp.youtube_url, ctx.work) + video, thumb = download_youtube(ctx.inp.youtube_url, ctx.work, + ctx.cfg, ctx.log) else: video = ctx.work / f"source_video{ctx.inp.video_path.suffix}" shutil.copyfile(ctx.inp.video_path, video) diff --git a/tests/test_acquire.py b/tests/test_acquire.py index 2058038..25cf43e 100644 --- a/tests/test_acquire.py +++ b/tests/test_acquire.py @@ -24,7 +24,7 @@ def test_local_video(tmp_path, monkeypatch): assert any("audio.wav" in str(a[-1]) for a in calls) def test_youtube(tmp_path, monkeypatch): - def fake_dl(url, dest): + def fake_dl(url, dest, cfg, log): video = dest / "source_video.webm" video.write_bytes(b"v") thumb = dest / "source_video.jpg" @@ -83,3 +83,95 @@ def test_classify_picks_largest_video_when_multiple(tmp_path): (tmp_path / "source_video.mp4").write_bytes(b"much larger contents here") video, _ = acquire._classify_downloads(tmp_path) assert video.name == "source_video.mp4" + +def test_classify_ignores_partials_left_by_a_failed_attempt(tmp_path): + (tmp_path / "source_video.webm").write_bytes(b"v") + (tmp_path / "source_video.f137.mp4.part").write_bytes(b"a much bigger partial") + (tmp_path / "source_video.f251.webm.ytdl").write_bytes(b"resume state") + video, _ = acquire._classify_downloads(tmp_path) + assert video.name == "source_video.webm" + +def test_ydl_opts_default_client_adds_no_player_client(tmp_path): + cfg = Config(whisper_url="http://x/v1") + opts = acquire._ydl_opts(cfg, tmp_path, "default", print) + assert "youtube" not in opts["extractor_args"] + assert opts["retries"] and opts["fragment_retries"] and opts["extractor_retries"] + +def test_ydl_opts_named_client_and_pot_provider(tmp_path): + cfg = Config(whisper_url="http://x/v1", + ytdlp_pot_provider_url="http://pot:4416", + ytdlp_cookies=tmp_path / "cookies.txt") + opts = acquire._ydl_opts(cfg, tmp_path, "tv", print) + assert opts["extractor_args"]["youtube"] == {"player_client": ["tv"]} + assert opts["extractor_args"]["youtubepot-bgutilhttp"] == { + "base_url": ["http://pot:4416"]} + assert opts["cookiefile"] == str(tmp_path / "cookies.txt") + +def _fake_yt_dlp(monkeypatch, behaviour): + """Install a stub yt_dlp whose download calls `behaviour(opts)`.""" + import sys + import types + seen = [] + + class FakeYoutubeDL: + def __init__(self, opts): + self.opts = opts + def __enter__(self): + return self + def __exit__(self, *exc): + return False + def extract_info(self, url, download): + seen.append(self.opts["extractor_args"].get("youtube")) + behaviour(self.opts) + + mod = types.ModuleType("yt_dlp") + mod.YoutubeDL = FakeYoutubeDL + mod.version = types.SimpleNamespace(__version__="2026.08.19") + monkeypatch.setitem(sys.modules, "yt_dlp", mod) + return seen + +def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch): + def behaviour(opts): + client = opts["extractor_args"].get("youtube") + if client is None: + raise RuntimeError("HTTP Error 403: Forbidden") + (tmp_path / "source_video.webm").write_bytes(b"v") + seen = _fake_yt_dlp(monkeypatch, behaviour) + cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"]) + logged = [] + video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg, + logged.append) + assert video.name == "source_video.webm" + assert thumb is None + assert seen == [None, {"player_client": ["tv"]}] + assert any("403" in m for m in logged) + +def test_download_reports_version_and_clients_when_all_fail(tmp_path, monkeypatch): + import pytest + from songtrace.pipeline import PipelineError + def behaviour(opts): + raise RuntimeError("HTTP Error 403: Forbidden") + _fake_yt_dlp(monkeypatch, behaviour) + cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"]) + with pytest.raises(PipelineError) as e: + acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None) + assert "2026.08.19" in str(e.value) + assert "default, tv" in str(e.value) + assert "403" in str(e.value) + +def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch): + seen = _fake_yt_dlp( + monkeypatch, lambda opts: (tmp_path / "source_video.mp4").write_bytes(b"v")) + cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[]) + acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None) + assert seen == [None] + +def test_ydl_log_drops_debug_lines(): + logged = [] + log = acquire._YdlLog(logged.append) + log.debug("[debug] internal detail") + log.debug("[youtube] Extracting URL") + log.warning("nsig extraction failed") + log.error("HTTP Error 403: Forbidden") + assert logged == ["[youtube] Extracting URL", "nsig extraction failed", + "HTTP Error 403: Forbidden"]