acquire: install yt-dlp's YouTube extras and retry across player clients

yt-dlp declares `dependencies = []` and puts everything in extras, so the
bare `yt-dlp` requirement installed it with no yt-dlp-ejs and no JavaScript
runtime. Without those it cannot solve YouTube's player challenges and falls
back to media URLs that YouTube rejects with HTTP 403, which is the failure
seen when adding a song. Require `yt-dlp[default,curl-cffi,deno]` instead;
the deno extra ships the JS runtime as a wheel, so the image needs no extra
apt packages.

Around that, make the stage survive the next time YouTube changes something:
the download is retried against each player client in `ytdlp_clients`
(default `["default", "tv"]`, the latter not needing a PO token) before
giving up, yt-dlp's retry counts are raised, and its own log lines are routed
into the job log so a PO-token demand can be told apart from a rate limit.
The final error names the running yt-dlp version, since a stale one is the
usual cause.

Cookies, TLS impersonation and a bgutil PO token provider URL are now config
fields, so an operator can work around a block without a code change.
`_classify_downloads` skips .part/.ytdl scratch files, which retries make far
more likely to be lying around.

README.md

@@ -51,7 +51,9 @@ $EDITOR config.toml   # set whisper_url at minimum
 
 See `config.example.toml` for every available field (`whisper_url`,
 `whisper_api_key`, `whisper_model`, `whisper_api`, `output_dir`, `work_dir`,
-`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`).
+`creator`, `demucs_model`, `crepe_model`, `device`, `bind_host`, `port`,
+`ytdlp_clients`, `ytdlp_cookies`, `ytdlp_impersonate`,
+`ytdlp_pot_provider_url`).
 
 `whisper_api` selects the wire format: `"openai"` (default) speaks the
 OpenAI-compatible `POST {whisper_url}/audio/transcriptions` API; `"asr-webservice"`
@@ -117,6 +119,46 @@ songtrace make --title "..." --artist "..." --lyrics lyrics.txt \
 Valid stages, in pipeline order: `acquire`, `separate`, `transcribe`,
 `align`, `pitch`, `notes`, `grid`, `package`.
 
+## YouTube download failures
+
+`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
+page but refused the media URL. In rough order of likelihood:
+
+1. **yt-dlp is stale.** YouTube changes its defences every few weeks and the
+   version baked into the image does not. Hotfix a running container, then
+   rebuild so it sticks:
+
+   ```sh
+   docker compose exec songtrace pip install -U 'yt-dlp[default,curl-cffi,deno]'
+   docker compose restart songtrace
+   docker compose build --pull songtrace   # permanent
+   ```
+
+2. **Missing JS runtime.** yt-dlp needs `yt-dlp-ejs` plus a JavaScript engine
+   to solve YouTube's player challenges; without them it silently falls back to
+   formats that 403. The `[default,curl-cffi,deno]` extras in `pyproject.toml`
+   cover this. Check with
+   `docker compose exec songtrace yt-dlp -v --simulate <url>` — the debug
+   header must show a `JS runtimes:` line that is not `none`.
+
+3. **The IP is rate-limited or flagged.** Datacenter and VPN ranges get
+   throttled hardest. Try from another network before changing config.
+
+4. **A PO token is required.** If the log says so explicitly, run
+   [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider)
+   as a sidecar, install its plugin into the image, and point
+   `ytdlp_pot_provider_url` at it.
+
+`ytdlp_clients` retries the download against each listed player client in
+turn, so a client that YouTube has broken is skipped automatically.
+`ytdlp_cookies` (a Netscape cookie jar, for age-gated videos) and
+`ytdlp_impersonate` (a browser TLS fingerprint, e.g. `"chrome"`) are the other
+two levers. Do not use a cookie jar from an account you care about — YouTube
+bans for automated access.
+
+Failed downloads leave the work dir intact, so re-running the job resumes
+rather than starting over.
+
 ## Regenerating the web UI screenshot
 
 `scripts/screenshot_webui.py` seeds the real app with a fixed set of jobs

config.example.toml

@@ -11,3 +11,10 @@ creator = "brad"
 # device = "cuda"
 bind_host = "0.0.0.0"
 # port = 8000
+
+# YouTube downloads. Each player client is tried in order until one works;
+# "default" means whatever yt-dlp currently picks for itself.
+# ytdlp_clients = ["default", "tv"]
+# ytdlp_cookies = "/config/youtube-cookies.txt"  # Netscape cookie jar
+# ytdlp_impersonate = "chrome"                   # TLS fingerprint to present
+# ytdlp_pot_provider_url = "http://bgutil-provider:4416"

pyproject.toml

@@ -5,7 +5,12 @@ description = "Generate UltraStar/Melody Mania karaoke songs from a video and re
 requires-python = ">=3.11"
 license = "AGPL-3.0-or-later"
 dependencies = [
-    "yt-dlp",
+    # yt-dlp declares no runtime deps of its own; the extras are what make
+    # YouTube work. "default" pulls yt-dlp-ejs (JS challenge solving) and the
+    # requests/urllib3 handlers, "deno" ships the JS runtime that executes it,
+    # and "curl-cffi" provides the --impersonate targets. Without them YouTube
+    # extraction degrades to HTTP 403 on the media URLs.
+    "yt-dlp[default,curl-cffi,deno]",
     "demucs",
     "torch",
     "torchaudio",

src/songtrace/config.py

@@ -1,6 +1,6 @@
 import os
 import tomllib
-from dataclasses import dataclass, fields
+from dataclasses import dataclass, field, fields
 from pathlib import Path
 
 
@@ -18,9 +18,15 @@ class Config:
     device: str = "cuda"
     bind_host: str = "127.0.0.1"
     port: int = 8000
+    # YouTube player clients to try, in order, until one yields a download.
+    # "default" means yt-dlp's own choice; see its extractor-args docs.
+    ytdlp_clients: list[str] = field(default_factory=lambda: ["default", "tv"])
+    ytdlp_cookies: Path | None = None
+    ytdlp_impersonate: str | None = None
+    ytdlp_pot_provider_url: str | None = None
 
 
-_PATH_FIELDS = {"output_dir", "work_dir"}
+_PATH_FIELDS = {"output_dir", "work_dir", "ytdlp_cookies"}
 
 
 def load_config(path: Path | None = None) -> Config:

src/songtrace/pipeline.py

@@ -45,6 +45,12 @@ def song_slug(inp: SongInput) -> str:
     name = f"{inp.artist} - {inp.title}"
     return name.replace("/", "").replace("\0", "")
 
+def read_warnings(work: Path) -> list[str]:
+    notes_file = work / "notes_sec.json"
+    if not notes_file.exists():
+        return []
+    return json.loads(notes_file.read_text()).get("warnings", [])
+
 def _real_stages() -> list[Stage]:
     from . import stages
     return stages.all_stages()
@@ -69,8 +75,5 @@ def run_pipeline(inp: SongInput, cfg: Config, log=print,
             continue
         log(f"[{stage.name}] running")
         stage.run(ctx)
-    warnings = []
-    notes_file = work / "notes_sec.json"
-    if notes_file.exists():
-        warnings = json.loads(notes_file.read_text()).get("warnings", [])
-    return PipelineResult(output_dir=cfg.output_dir / song_slug(inp), warnings=warnings)
+    return PipelineResult(output_dir=cfg.output_dir / song_slug(inp),
+                          warnings=read_warnings(work))

src/songtrace/stages/acquire.py

@@ -2,14 +2,18 @@ import json
 import shutil
 from pathlib import Path
 
+from ..config import Config
 from ..media import ffmpeg
 from ..pipeline import Ctx, PipelineError, Stage
 
 IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp", ".avif", ".gif"}
+# yt-dlp's in-progress scratch files, which a failed attempt leaves behind.
+PARTIAL_EXTENSIONS = {".part", ".ytdl", ".temp"}
 
 
 def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
-    files = list(dest.glob("source_video.*"))
+    files = [p for p in dest.glob("source_video.*")
+             if p.suffix.lower() not in PARTIAL_EXTENSIONS]
     thumbs = [p for p in files if p.suffix.lower() in IMAGE_EXTENSIONS]
     videos = [p for p in files if p.suffix.lower() not in IMAGE_EXTENSIONS]
     if not videos:
@@ -18,17 +22,67 @@ def _classify_downloads(dest: Path) -> tuple[Path, Path | None]:
     return video, thumbs[0] if thumbs else None
 
 
-def download_youtube(url: str, dest: Path) -> tuple[Path, Path | None]:
-    import yt_dlp
+class _YdlLog:
+    """Routes yt-dlp's own messages into the job log, minus progress noise."""
+
+    def __init__(self, log):
+        self._log = log
+
+    def debug(self, msg: str) -> None:
+        if not msg.startswith("[debug] "):
+            self._log(msg)
+
+    def warning(self, msg: str) -> None:
+        self._log(msg)
+
+    error = warning
+
+
+def _ydl_opts(cfg: Config, dest: Path, client: str, log) -> dict:
+    extractor_args: dict[str, dict[str, list[str]]] = {}
+    if client != "default":
+        extractor_args["youtube"] = {"player_client": [client]}
+    if cfg.ytdlp_pot_provider_url:
+        extractor_args["youtubepot-bgutilhttp"] = {
+            "base_url": [cfg.ytdlp_pot_provider_url]}
     opts = {
         "outtmpl": str(dest / "source_video.%(ext)s"),
         "format": "bestvideo*+bestaudio/best",
         "writethumbnail": True,
         "noplaylist": True,
+        "noprogress": True,
+        "logger": _YdlLog(log),
+        "retries": 10,
+        "fragment_retries": 10,
+        "extractor_retries": 5,
+        "extractor_args": extractor_args,
     }
-    with yt_dlp.YoutubeDL(opts) as ydl:
-        ydl.extract_info(url, download=True)
-    return _classify_downloads(dest)
+    if cfg.ytdlp_cookies:
+        opts["cookiefile"] = str(cfg.ytdlp_cookies)
+    if cfg.ytdlp_impersonate:
+        from yt_dlp.networking.impersonate import ImpersonateTarget
+        opts["impersonate"] = ImpersonateTarget.from_str(cfg.ytdlp_impersonate)
+    return opts
+
+
+def download_youtube(url: str, dest: Path, cfg: Config,
+                     log=print) -> tuple[Path, Path | None]:
+    import yt_dlp
+    clients = list(cfg.ytdlp_clients) or ["default"]
+    last: Exception | None = None
+    for client in clients:
+        try:
+            with yt_dlp.YoutubeDL(_ydl_opts(cfg, dest, client, log)) as ydl:
+                ydl.extract_info(url, download=True)
+            return _classify_downloads(dest)
+        except Exception as e:
+            last = e
+            log(f"yt-dlp player_client={client} failed: {e}")
+    raise PipelineError(
+        f"yt-dlp {yt_dlp.version.__version__} could not download {url} after "
+        f"trying player_client={', '.join(clients)}. Last error: {last}. "
+        "YouTube blocks change often: upgrade yt-dlp, then see the yt-dlp "
+        "options in config.toml (cookies, impersonation, PO token provider).")
 
 
 def _run(ctx: Ctx) -> None:
@@ -36,7 +90,8 @@ def _run(ctx: Ctx) -> None:
         raise PipelineError("provide exactly one of youtube_url / video_path")
     thumb = None
     if ctx.inp.youtube_url:
-        video, thumb = download_youtube(ctx.inp.youtube_url, ctx.work)
+        video, thumb = download_youtube(ctx.inp.youtube_url, ctx.work,
+                                       ctx.cfg, ctx.log)
     else:
         video = ctx.work / f"source_video{ctx.inp.video_path.suffix}"
         shutil.copyfile(ctx.inp.video_path, video)

src/songtrace/static/index.html

@@ -776,6 +776,7 @@ async function refreshJobs() {
   renderJobs(jobs);
   renderNowBar(jobs);
   if (selectedId) refreshDetail();
+  else clearDetail();
   schedulePoll(jobs);
 }
 
@@ -999,6 +1000,21 @@ function renderOutputPath(job) {
   outputPathDiv.replaceChildren(wrap);
 }
 
+function clearDetail() {
+  reviewTitle.replaceChildren();
+  reviewError.hidden = true;
+  reviewError.textContent = "";
+  document.getElementById("warnings").replaceChildren(
+    emptyNote("no songs queued yet — nothing to review"));
+  outputPathDiv.replaceChildren();
+  reviewActions.replaceChildren();
+  delete reviewActions.dataset.jobId;
+  reviewLineage.replaceChildren();
+  reviewLineage.hidden = true;
+  delete reviewLineage.dataset.jobId;
+  detail.textContent = "no songs queued yet — nothing to log";
+}
+
 function setFormMode(inp) {
   formHeading.textContent = rebuildOf ? "REBUILD" : "NEW SONG";
   submitButton.textContent = rebuildOf ? "REBUILD SONG" : "CREATE SONG";
@@ -1045,21 +1061,60 @@ async function rerunJob(id) {
   refreshJobs();
 }
 
+async function rebuildFullJob(id) {
+  formError.textContent = "";
+  const resp = await fetch(`/api/jobs/${id}/rebuild-full`, { method: "POST" });
+  if (!resp.ok) {
+    const body = await resp.json().catch(() => ({}));
+    formError.textContent = body.detail || `error: ${resp.status}`;
+    return;
+  }
+  selectedId = (await resp.json()).id;
+  refreshJobs();
+}
+
+async function deleteJob(id) {
+  if (!confirm("Delete this song? This removes all its files and cannot be undone.")) return;
+  formError.textContent = "";
+  const resp = await fetch(`/api/jobs/${id}`, { method: "DELETE" });
+  if (!resp.ok) {
+    const body = await resp.json().catch(() => ({}));
+    formError.textContent = body.detail || `error: ${resp.status}`;
+    return;
+  }
+  if (selectedId === id) selectedId = null;
+  refreshJobs();
+}
+
+const ACTION_LABELS = {
+  rerun: "RERUN",
+  edit: "EDIT & REBUILD",
+  "rebuild-full": "REBUILD FROM SCRATCH",
+  delete: "DELETE",
+};
+
+const ACTION_HANDLERS = {
+  rerun: rerunJob,
+  edit: startRebuild,
+  "rebuild-full": rebuildFullJob,
+  delete: deleteJob,
+};
+
 function actionButton(action, jobId) {
   const button = document.createElement("button");
   button.type = "button";
   button.className = "action";
   button.dataset.action = action;
-  button.textContent = action === "rerun" ? "RERUN" : "EDIT & REBUILD";
-  button.addEventListener("click",
-    () => action === "rerun" ? rerunJob(jobId) : startRebuild(jobId));
+  button.textContent = ACTION_LABELS[action];
+  button.addEventListener("click", () => ACTION_HANDLERS[action](jobId));
   return button;
 }
 
 function renderActions(job) {
-  // no controls mid-run: both endpoints answer 409 until the job settles
-  const wanted = job.status === "failed" ? ["rerun", "edit"]
-               : job.status === "done" ? ["edit"] : [];
+  // no controls mid-run: every endpoint answers 409 until the job settles
+  const doneActions = ["edit", "rebuild-full", "delete"];
+  const wanted = job.status === "failed" ? ["rerun", ...doneActions]
+               : job.status === "done" ? doneActions : [];
   const shown = [...reviewActions.children].map(b => b.dataset.action);
   // rebuilding an unchanged row would detach a focused action on every poll
   if (reviewActions.dataset.jobId === job.id && shown.join() === wanted.join()) return;

src/songtrace/web.py

@@ -6,7 +6,7 @@ import shutil
 import threading
 import time
 import uuid
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from pathlib import Path
 
 from fastapi import FastAPI, File, Form, HTTPException, UploadFile
@@ -17,8 +17,8 @@ from fastapi.staticfiles import StaticFiles
 from . import rebuild as rebuild_policy
 from .config import Config
 from .lyrics import normalize_lyrics
-from .pipeline import (STAGE_ORDER, PipelineError, SongInput, run_pipeline,
-                       song_slug)
+from .pipeline import (STAGE_ORDER, PipelineError, SongInput, read_warnings,
+                       run_pipeline, song_slug)
 
 STATIC = Path(__file__).parent / "static"
 STAGE_LINE = re.compile(r"^\[(\w+)\] (?:running|cached, skipping)")
@@ -37,6 +37,7 @@ class Job:
     error: str | None = None
     inp: SongInput | None = None
     rebuilt_from: str | None = None
+    force_stage: str | None = None
     created_at: float = field(default_factory=time.time)
     started_at: float | None = None
     finished_at: float | None = None
@@ -93,6 +94,35 @@ def _cleanup_uploads(cfg: Config, job_id: str, video_path: Path | None) -> None:
             pass
 
 
+def _discover_jobs(cfg: Config) -> list[Job]:
+    discovered = []
+    if not cfg.work_dir.exists():
+        return discovered
+    for sidecar in cfg.work_dir.glob("*/input.json"):
+        try:
+            data = json.loads(sidecar.read_text())
+            for key in ("video_path", "cover_path"):
+                if data.get(key) is not None:
+                    data[key] = Path(data[key])
+            inp = SongInput(**data)
+            slug = song_slug(inp)
+            out = cfg.output_dir / slug
+            done = out.exists() and any(out.iterdir())
+            mtime = sidecar.stat().st_mtime
+            warnings = read_warnings(sidecar.parent) if done else []
+        except (OSError, ValueError, TypeError):
+            continue
+        discovered.append(Job(
+            id=uuid.uuid4().hex, title=inp.title, artist=inp.artist, inp=inp,
+            status="done" if done else "failed",
+            output_dir=str(out) if done else None,
+            warnings=warnings,
+            error=None if done else "interrupted before completion",
+            created_at=mtime, finished_at=mtime))
+    discovered.sort(key=lambda j: j.created_at)
+    return discovered
+
+
 def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
     app = FastAPI()
     app.mount("/static", StaticFiles(directory=STATIC), name="static")
@@ -101,8 +131,15 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
     lock = threading.Lock()
     todo: queue.Queue[str] = queue.Queue()
 
+    for job in _discover_jobs(cfg):
+        jobs[job.id] = job
+        order.append(job.id)
+
     def _prepare_rebuild(job: Job, log) -> str | None:
         """Relocate cached work and report the stage this rebuild restarts from."""
+        if job.force_stage:
+            log(f"[rebuild] restarting from {job.force_stage} (forced rebuild from scratch)")
+            return job.force_stage
         if not job.rebuilt_from:
             return None
         with lock:
@@ -130,6 +167,15 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
         shutil.rmtree(old, ignore_errors=True)
         log(f"[rebuild] removed superseded {old.name}")
 
+    def _write_sidecar(job: Job) -> None:
+        work = cfg.work_dir / song_slug(job.inp)
+        work.mkdir(parents=True, exist_ok=True)
+        data = asdict(job.inp)
+        for key in ("video_path", "cover_path"):
+            if data[key] is not None:
+                data[key] = str(data[key])
+        (work / "input.json").write_text(json.dumps(data))
+
     def worker():
         while True:
             job_id = todo.get()
@@ -146,6 +192,7 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
 
             try:
                 force_from = _prepare_rebuild(job, log)
+                _write_sidecar(job)
                 result = runner(job.inp, cfg, log=log, force_from=force_from)
                 with lock:
                     job.output_dir = str(result.output_dir)
@@ -166,10 +213,10 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
 
     threading.Thread(target=worker, daemon=True).start()
 
-    def _enqueue(job_id: str, inp: SongInput,
-                 rebuilt_from: str | None = None) -> dict:
+    def _enqueue(job_id: str, inp: SongInput, rebuilt_from: str | None = None,
+                 force_stage: str | None = None) -> dict:
         job = Job(id=job_id, title=inp.title, artist=inp.artist, inp=inp,
-                  rebuilt_from=rebuilt_from)
+                  rebuilt_from=rebuilt_from, force_stage=force_stage)
         with lock:
             jobs[job_id] = job
             order.append(job_id)
@@ -218,6 +265,12 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
         parent = _finished_job(job_id)
         return _enqueue(uuid.uuid4().hex, parent.inp, rebuilt_from=job_id)
 
+    @app.post("/api/jobs/{job_id}/rebuild-full", status_code=201)
+    def rebuild_full(job_id: str):
+        parent = _finished_job(job_id)
+        return _enqueue(uuid.uuid4().hex, parent.inp, rebuilt_from=job_id,
+                        force_stage=STAGE_ORDER[0])
+
     def _require_source(parent_inp: SongInput, inp: SongInput,
                         force_from: str | None) -> None:
         """Refuse a rebuild that will rerun acquire against an upload we no longer have."""
@@ -305,6 +358,24 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
                 "video_filename": inp.video_path.name if inp.video_path else None,
                 "cover_filename": inp.cover_path.name if inp.cover_path else None}
 
+    @app.delete("/api/jobs/{job_id}", status_code=204)
+    def delete_job(job_id: str):
+        with lock:
+            job = jobs.get(job_id)
+            if job is None:
+                raise HTTPException(404)
+            if job.status in ("queued", "running"):
+                raise HTTPException(409, "job is still in progress")
+            slug = song_slug(job.inp)
+            if any(other.status in ("queued", "running") and song_slug(other.inp) == slug
+                   for other_id, other in jobs.items() if other_id != job_id):
+                raise HTTPException(409, "another job for this song is in progress")
+            jobs.pop(job_id, None)
+            if job_id in order:
+                order.remove(job_id)
+        shutil.rmtree(cfg.work_dir / slug, ignore_errors=True)
+        shutil.rmtree(cfg.output_dir / slug, ignore_errors=True)
+
     @app.get("/api/jobs/{job_id}/review")
     def get_review(job_id: str):
         job = _finished_job(job_id)

tests/test_acquire.py

@@ -24,7 +24,7 @@ def test_local_video(tmp_path, monkeypatch):
     assert any("audio.wav" in str(a[-1]) for a in calls)
 
 def test_youtube(tmp_path, monkeypatch):
-    def fake_dl(url, dest):
+    def fake_dl(url, dest, cfg, log):
         video = dest / "source_video.webm"
         video.write_bytes(b"v")
         thumb = dest / "source_video.jpg"
@@ -83,3 +83,95 @@ def test_classify_picks_largest_video_when_multiple(tmp_path):
     (tmp_path / "source_video.mp4").write_bytes(b"much larger contents here")
     video, _ = acquire._classify_downloads(tmp_path)
     assert video.name == "source_video.mp4"
+
+def test_classify_ignores_partials_left_by_a_failed_attempt(tmp_path):
+    (tmp_path / "source_video.webm").write_bytes(b"v")
+    (tmp_path / "source_video.f137.mp4.part").write_bytes(b"a much bigger partial")
+    (tmp_path / "source_video.f251.webm.ytdl").write_bytes(b"resume state")
+    video, _ = acquire._classify_downloads(tmp_path)
+    assert video.name == "source_video.webm"
+
+def test_ydl_opts_default_client_adds_no_player_client(tmp_path):
+    cfg = Config(whisper_url="http://x/v1")
+    opts = acquire._ydl_opts(cfg, tmp_path, "default", print)
+    assert "youtube" not in opts["extractor_args"]
+    assert opts["retries"] and opts["fragment_retries"] and opts["extractor_retries"]
+
+def test_ydl_opts_named_client_and_pot_provider(tmp_path):
+    cfg = Config(whisper_url="http://x/v1",
+                 ytdlp_pot_provider_url="http://pot:4416",
+                 ytdlp_cookies=tmp_path / "cookies.txt")
+    opts = acquire._ydl_opts(cfg, tmp_path, "tv", print)
+    assert opts["extractor_args"]["youtube"] == {"player_client": ["tv"]}
+    assert opts["extractor_args"]["youtubepot-bgutilhttp"] == {
+        "base_url": ["http://pot:4416"]}
+    assert opts["cookiefile"] == str(tmp_path / "cookies.txt")
+
+def _fake_yt_dlp(monkeypatch, behaviour):
+    """Install a stub yt_dlp whose download calls `behaviour(opts)`."""
+    import sys
+    import types
+    seen = []
+
+    class FakeYoutubeDL:
+        def __init__(self, opts):
+            self.opts = opts
+        def __enter__(self):
+            return self
+        def __exit__(self, *exc):
+            return False
+        def extract_info(self, url, download):
+            seen.append(self.opts["extractor_args"].get("youtube"))
+            behaviour(self.opts)
+
+    mod = types.ModuleType("yt_dlp")
+    mod.YoutubeDL = FakeYoutubeDL
+    mod.version = types.SimpleNamespace(__version__="2026.08.19")
+    monkeypatch.setitem(sys.modules, "yt_dlp", mod)
+    return seen
+
+def test_download_falls_through_to_the_next_client(tmp_path, monkeypatch):
+    def behaviour(opts):
+        client = opts["extractor_args"].get("youtube")
+        if client is None:
+            raise RuntimeError("HTTP Error 403: Forbidden")
+        (tmp_path / "source_video.webm").write_bytes(b"v")
+    seen = _fake_yt_dlp(monkeypatch, behaviour)
+    cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
+    logged = []
+    video, thumb = acquire.download_youtube("https://yt/x", tmp_path, cfg,
+                                            logged.append)
+    assert video.name == "source_video.webm"
+    assert thumb is None
+    assert seen == [None, {"player_client": ["tv"]}]
+    assert any("403" in m for m in logged)
+
+def test_download_reports_version_and_clients_when_all_fail(tmp_path, monkeypatch):
+    import pytest
+    from songtrace.pipeline import PipelineError
+    def behaviour(opts):
+        raise RuntimeError("HTTP Error 403: Forbidden")
+    _fake_yt_dlp(monkeypatch, behaviour)
+    cfg = Config(whisper_url="http://x/v1", ytdlp_clients=["default", "tv"])
+    with pytest.raises(PipelineError) as e:
+        acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
+    assert "2026.08.19" in str(e.value)
+    assert "default, tv" in str(e.value)
+    assert "403" in str(e.value)
+
+def test_download_with_no_configured_clients_still_tries_default(tmp_path, monkeypatch):
+    seen = _fake_yt_dlp(
+        monkeypatch, lambda opts: (tmp_path / "source_video.mp4").write_bytes(b"v"))
+    cfg = Config(whisper_url="http://x/v1", ytdlp_clients=[])
+    acquire.download_youtube("https://yt/x", tmp_path, cfg, lambda m: None)
+    assert seen == [None]
+
+def test_ydl_log_drops_debug_lines():
+    logged = []
+    log = acquire._YdlLog(logged.append)
+    log.debug("[debug] internal detail")
+    log.debug("[youtube] Extracting URL")
+    log.warning("nsig extraction failed")
+    log.error("HTTP Error 403: Forbidden")
+    assert logged == ["[youtube] Extracting URL", "nsig extraction failed",
+                      "HTTP Error 403: Forbidden"]

tests/test_pipeline.py

@@ -1,7 +1,7 @@
 import json
 from pathlib import Path
 from songtrace.config import Config
-from songtrace.pipeline import SongInput, Stage, run_pipeline, song_slug
+from songtrace.pipeline import SongInput, Stage, read_warnings, run_pipeline, song_slug
 
 def make_cfg(tmp_path):
     return Config(whisper_url="http://x/v1", output_dir=tmp_path / "out",
@@ -46,3 +46,10 @@ def test_warnings_surfaced(tmp_path):
     st = Stage(name="n", outputs=lambda ctx: [ctx.work / "n.done"], run=run)
     result = run_pipeline(inp, cfg, log=lambda m: None, stages=[st])
     assert result.warnings == ["w1"]
+
+def test_read_warnings_is_empty_without_a_notes_file(tmp_path):
+    assert read_warnings(tmp_path) == []
+
+def test_read_warnings_reads_the_notes_file(tmp_path):
+    (tmp_path / "notes_sec.json").write_text(json.dumps({"warnings": ["w1", "w2"]}))
+    assert read_warnings(tmp_path) == ["w1", "w2"]

tests/test_web.py

@@ -1,6 +1,7 @@
 import io
 import json
 import re
+import shutil
 import threading
 import time
 
@@ -601,6 +602,44 @@ def _rebuild_flaws(html):
     return flaws
 
 
+def _rebuild_full_and_delete_flaws(html):
+    flaws = []
+    actions = _fn_body(html, "renderActions")
+    for action in ('"rebuild-full"', '"delete"'):
+        if action not in actions:
+            flaws.append(f"{action} is not offered from renderActions")
+
+    rf = _fn_body(html, "rebuildFullJob")
+    if "/rebuild-full" not in rf or 'method: "POST"' not in rf:
+        flaws.append("rebuildFullJob does not POST to /rebuild-full")
+
+    d = _fn_body(html, "deleteJob")
+    confirm_idx = d.find("confirm(")
+    fetch_idx = d.find("fetch(")
+    if confirm_idx == -1:
+        flaws.append("deleteJob does not confirm before deleting")
+    elif fetch_idx != -1 and confirm_idx > fetch_idx:
+        flaws.append("deleteJob fetches before confirming")
+    if 'method: "DELETE"' not in d:
+        flaws.append("deleteJob does not send a DELETE request")
+    return flaws
+
+
+def test_rebuild_full_and_delete_controls_are_wired_up():
+    assert _rebuild_full_and_delete_flaws(_index_html()) == []
+
+
+def _empty_state_flaws(html):
+    refresh = _fn_body(html, "refreshJobs")
+    if "clearDetail()" not in refresh:
+        return ["refreshJobs never resets the panel when no job is selected"]
+    return []
+
+
+def test_review_panel_resets_when_no_job_is_selected():
+    assert _empty_state_flaws(_index_html()) == []
+
+
 # each mutation is a plausible regression; the checker above must reject all of them
 MUTATIONS = [
     (_focus_flaws, "queue: card moved unconditionally",
@@ -629,6 +668,14 @@ MUTATIONS = [
     (_rebuild_flaws, "rebuild: action row rebuilt on every poll",
      "if (reviewActions.dataset.jobId === job.id && shown.join() === wanted.join()) return;",
      ""),
+    (_rebuild_full_and_delete_flaws, "delete: confirmation dialog removed",
+     'if (!confirm("Delete this song? This removes all its files and cannot be undone.")) return;',
+     ""),
+    (_rebuild_full_and_delete_flaws, "rebuild-full: no longer offered on a done job",
+     '["edit", "rebuild-full", "delete"]', '["edit", "delete"]'),
+    (_empty_state_flaws, "empty state: clearDetail no longer wired into refreshJobs",
+     "if (selectedId) refreshDetail();\n  else clearDetail();",
+     "if (selectedId) refreshDetail();"),
 ]
 
 
@@ -846,7 +893,7 @@ def test_rebuild_renames_the_work_dir_on_a_retitle(tmp_path):
     seen = []
     client, cfg = make_client(tmp_path, capture_runner_factory(seen))
     parent = make_done_job(client)
-    (cfg.work_dir / "A - T").mkdir(parents=True)
+    (cfg.work_dir / "A - T").mkdir(parents=True, exist_ok=True)
     (cfg.work_dir / "A - T" / "audio.wav").write_bytes(b"x")
     child = client.post(f"/api/jobs/{parent}/rebuild",
                         data=rebuild_form(title="T2")).json()["id"]
@@ -859,7 +906,7 @@ def test_rebuild_renames_the_work_dir_on_a_retitle(tmp_path):
 def test_rebuild_fails_the_job_on_a_work_dir_collision(tmp_path):
     client, cfg = make_client(tmp_path, ok_runner)
     parent = make_done_job(client)
-    (cfg.work_dir / "A - T").mkdir(parents=True)
+    (cfg.work_dir / "A - T").mkdir(parents=True, exist_ok=True)
     (cfg.work_dir / "A - T2").mkdir(parents=True)
     child = client.post(f"/api/jobs/{parent}/rebuild",
                         data=rebuild_form(title="T2")).json()["id"]
@@ -887,7 +934,7 @@ def test_rebuild_inherits_the_parents_source(tmp_path):
                         files={"video": ("v.mp4", io.BytesIO(b"data"), "video/mp4")})
     parent = resp.json()["id"]
     wait_status(client, parent, "done")
-    (cfg.work_dir / "A - T").mkdir(parents=True)
+    (cfg.work_dir / "A - T").mkdir(parents=True, exist_ok=True)
     for name in ("meta.json", "audio.wav"):
         (cfg.work_dir / "A - T" / name).write_bytes(b"x")
     child = client.post(f"/api/jobs/{parent}/rebuild",
@@ -973,6 +1020,59 @@ def test_rebuild_of_an_unknown_job_is_404(tmp_path):
     assert client.post("/api/jobs/nope/rebuild", data=rebuild_form()).status_code == 404
 
 
+def test_rebuild_full_forces_every_stage_from_the_first(tmp_path):
+    seen = []
+    client, _ = make_client(tmp_path, capture_runner_factory(seen))
+    parent = make_done_job(client)
+    child = client.post(f"/api/jobs/{parent}/rebuild-full").json()["id"]
+    wait_status(client, child, "done")
+    assert seen[1]["force_from"] == STAGE_ORDER[0]
+    assert seen[1]["inp"] == seen[0]["inp"]
+
+
+def test_rebuild_full_creates_a_linked_job(tmp_path):
+    client, _ = make_client(tmp_path, ok_runner)
+    parent = make_done_job(client)
+    resp = client.post(f"/api/jobs/{parent}/rebuild-full")
+    assert resp.status_code == 201
+    child = resp.json()["id"]
+    assert child != parent
+    job = wait_status(client, child, "done")
+    assert job["rebuilt_from"] == parent
+
+
+def test_rebuild_full_logs_that_it_is_a_forced_rebuild(tmp_path):
+    client, _ = make_client(tmp_path, ok_runner)
+    parent = make_done_job(client)
+    child = client.post(f"/api/jobs/{parent}/rebuild-full").json()["id"]
+    job = wait_status(client, child, "done")
+    rebuild_lines = [l for l in job["log"] if l.startswith("[rebuild]")]
+    assert len(rebuild_lines) == 1
+    assert STAGE_ORDER[0] in rebuild_lines[0]
+
+
+def test_rebuild_full_of_an_unknown_job_is_404(tmp_path):
+    client, _ = make_client(tmp_path, ok_runner)
+    assert client.post("/api/jobs/nope/rebuild-full").status_code == 404
+
+
+def test_rebuild_full_of_a_running_job_is_409(tmp_path):
+    release = threading.Event()
+
+    def blocking(inp, cfg, log=print, force_from=None, stages=None):
+        release.wait(5)
+        return PipelineResult(output_dir=cfg.output_dir / "X")
+
+    client, _ = make_client(tmp_path, blocking)
+    parent = client.post("/api/jobs", data=rebuild_form()).json()["id"]
+    try:
+        wait_status(client, parent, "running")
+        assert client.post(f"/api/jobs/{parent}/rebuild-full").status_code == 409
+    finally:
+        release.set()
+    wait_status(client, parent, "done")
+
+
 def slug_runner(inp, cfg, log=print, force_from=None, stages=None):
     out = cfg.output_dir / song_slug(inp)
     out.mkdir(parents=True, exist_ok=True)
@@ -1021,6 +1121,76 @@ def test_rebuild_of_a_failed_parent_removes_nothing(tmp_path):
     assert (cfg.output_dir / "A - T" / "keep.txt").exists()
 
 
+def test_delete_removes_work_and_output_and_the_job(tmp_path):
+    client, cfg = make_client(tmp_path, slug_runner)
+    job_id = make_done_job(client)
+    (cfg.work_dir / "A - T").mkdir(parents=True, exist_ok=True)
+    (cfg.work_dir / "A - T" / "audio.wav").write_bytes(b"x")
+    assert (cfg.output_dir / "A - T").exists()
+
+    resp = client.delete(f"/api/jobs/{job_id}")
+
+    assert resp.status_code == 204
+    assert not (cfg.work_dir / "A - T").exists()
+    assert not (cfg.output_dir / "A - T").exists()
+    assert client.get(f"/api/jobs/{job_id}").status_code == 404
+    assert job_id not in [j["id"] for j in client.get("/api/jobs").json()]
+
+
+def test_delete_tolerates_a_work_dir_that_is_already_gone(tmp_path):
+    client, cfg = make_client(tmp_path, slug_runner)
+    job_id = make_done_job(client)
+    shutil.rmtree(cfg.work_dir / "A - T")
+    resp = client.delete(f"/api/jobs/{job_id}")
+    assert resp.status_code == 204
+
+
+def test_delete_of_an_unknown_job_is_404(tmp_path):
+    client, _ = make_client(tmp_path, ok_runner)
+    assert client.delete("/api/jobs/nope").status_code == 404
+
+
+def test_delete_of_a_running_job_is_409(tmp_path):
+    release = threading.Event()
+
+    def blocking(inp, cfg, log=print, force_from=None, stages=None):
+        release.wait(5)
+        return PipelineResult(output_dir=cfg.output_dir / "X")
+
+    client, _ = make_client(tmp_path, blocking)
+    parent = client.post("/api/jobs", data=rebuild_form()).json()["id"]
+    try:
+        wait_status(client, parent, "running")
+        assert client.delete(f"/api/jobs/{parent}").status_code == 409
+    finally:
+        release.set()
+    wait_status(client, parent, "done")
+    assert client.get(f"/api/jobs/{parent}").status_code == 200
+
+
+def test_delete_of_a_job_with_an_active_rebuild_is_409(tmp_path):
+    """A done job's slug must not be deleted out from under a rebuild in flight."""
+    release = threading.Event()
+    calls = []
+
+    def runner(inp, cfg, log=print, force_from=None, stages=None):
+        calls.append(1)
+        if len(calls) > 1:
+            release.wait(5)
+        return PipelineResult(output_dir=cfg.output_dir / "X")
+
+    client, _ = make_client(tmp_path, runner)
+    parent = make_done_job(client)
+    child = client.post(f"/api/jobs/{parent}/rebuild-full").json()["id"]
+    try:
+        wait_status(client, child, "running")
+        assert client.delete(f"/api/jobs/{parent}").status_code == 409
+    finally:
+        release.set()
+    wait_status(client, child, "done")
+    assert client.get(f"/api/jobs/{parent}").status_code == 200
+
+
 def test_a_plain_job_never_removes_anything(tmp_path):
     client, cfg = make_client(tmp_path, slug_runner)
     make_done_job(client)
@@ -1030,6 +1200,54 @@ def test_a_plain_job_never_removes_anything(tmp_path):
     assert stray.exists()
 
 
+def test_discover_jobs_ignores_a_work_dir_without_a_sidecar(tmp_path):
+    from songtrace.web import _discover_jobs
+    cfg = Config(whisper_url="u", work_dir=tmp_path / "work", output_dir=tmp_path / "out")
+    (cfg.work_dir / "A - T").mkdir(parents=True)
+    assert _discover_jobs(cfg) == []
+
+
+def test_discover_jobs_reconstructs_a_completed_song(tmp_path):
+    from songtrace.web import _discover_jobs
+    cfg = Config(whisper_url="u", work_dir=tmp_path / "work", output_dir=tmp_path / "out")
+    client = TestClient(create_app(cfg, runner=slug_runner))
+    make_done_job(client)
+
+    discovered = _discover_jobs(cfg)
+
+    assert len(discovered) == 1
+    job = discovered[0]
+    assert job.title == "T" and job.artist == "A" and job.status == "done"
+    assert job.output_dir == str(cfg.output_dir / "A - T")
+    assert job.error is None
+
+
+def test_discover_jobs_reconstructs_a_song_whose_output_is_missing_as_failed(tmp_path):
+    from songtrace.web import _discover_jobs
+    cfg = Config(whisper_url="u", work_dir=tmp_path / "work", output_dir=tmp_path / "out")
+    client = TestClient(create_app(cfg, runner=slug_runner))
+    make_done_job(client)
+    shutil.rmtree(cfg.output_dir / "A - T")
+
+    discovered = _discover_jobs(cfg)
+
+    assert len(discovered) == 1
+    assert discovered[0].status == "failed"
+    assert discovered[0].error == "interrupted before completion"
+    assert discovered[0].output_dir is None
+
+
+def test_discovered_jobs_are_visible_from_a_fresh_app(tmp_path):
+    cfg = Config(whisper_url="u", work_dir=tmp_path / "work", output_dir=tmp_path / "out")
+    client1 = TestClient(create_app(cfg, runner=slug_runner))
+    make_done_job(client1)
+
+    client2 = TestClient(create_app(cfg, runner=slug_runner))
+    jobs = client2.get("/api/jobs").json()
+    assert len(jobs) == 1
+    assert jobs[0]["title"] == "T" and jobs[0]["status"] == "done"
+
+
 def review_runner_factory(notes, dropped_lines=None, missing_spans=None, warnings=None,
                           words=None):
     def runner(inp, cfg, log=print, force_from=None, stages=None):