webui: play a finished song against its review

Checking whether the notes line up meant packaging the song, finding it in
the game and starting it there. The review panel now plays the song folder
itself: a transport above the lyrics, the line being sung highlighted, words
lighting up as they are reached, and a playhead sweeping each line against
its notes. Clicking a line or a word plays from there.

Playback is modelled on the game rather than on whatever is convenient: #MP3
is the audio and the clock, #VIDEO is muted background slaved to it, and the
times are the same seconds the notes were generated from. A word that lights
up late here is late in the game.

package now strips the audio track from the video it writes. It is the same
recording as the #MP3, so a player that does not mute the video plays both a
few frames apart, which is heard as an echo. The compatible-video branch
remuxes with -c copy -an rather than copying the file, so it still does not
re-encode. The browser player mutes the video as well, which covers song
folders built before this.

A new /api/jobs/{id}/media/{audio,video,cover} serves what the song's own
#MP3/#VIDEO/#COVER headers name, refusing anything resolving outside the song
folder. Range requests come from FileResponse, so seeking works.

Also, per review: CREATE SONG moves out of the form and into the top of the
queue, where it is always available, and a successful submission keeps the
fields it sent instead of emptying them.

README.md

@@ -34,6 +34,26 @@ printing the low-confidence regions it found (e.g.
 editor such as [Yass](https://www.yass-along.com/) or
 [USDX](https://usdx.eu/) and touch those spots up before playing.
 
+## Reviewing a song
+
+The web UI plays the finished song folder back against the review, so you can
+check the timing without loading it into a game first. Pick a song in the
+queue and the review panel gets the packaged video with a transport above the
+lyrics; as it plays, the line being sung is highlighted, words light up as
+they are reached, and a playhead sweeps each line against its notes. Click any
+line or word to play from there.
+
+Playback is modelled on the game: the `#MP3` is the audio and the clock, and
+the `#VIDEO` is muted background slaved to it. It also uses the same seconds
+the notes were generated from, so a word that lights up late in the browser is
+late in the game too.
+
+The packaged video has its audio track stripped for the same reason. It is the
+same recording as the `#MP3`, so a player that does not mute the video plays
+both at once, a few frames apart, which is heard as an echo. Songs built
+before this change still have it — rebuild them, or check with
+`ffprobe <song>.mp4` for an audio stream.
+
 ## Configuration
 
 Config is TOML. Resolution order:
@@ -168,7 +188,9 @@ rather than starting over.
 ## Regenerating the web UI screenshot
 
 `scripts/screenshot_webui.py` seeds the real app with a fixed set of jobs
-covering every status and screenshots it to `docs/images/webui.png`:
+covering every status and screenshots it to `docs/images/webui.png`. It uses
+`ffmpeg` to give the selected song a playable folder so the shot includes the
+player; without `ffmpeg` on `PATH` it still runs, just without it.
 
 ```sh
 pip install -e '.[dev]'

scripts/screenshot_webui.py

@@ -7,6 +7,8 @@ One-time setup: pip install -e '.[dev]' && playwright install chromium
 Run: python scripts/screenshot_webui.py
 """
 import json
+import shutil
+import subprocess
 import tempfile
 import threading
 import time
@@ -110,6 +112,7 @@ SONGS = [
     ("Vampire in the Corner", "Magdalena Bay", "queued", {}),
 ]
 SELECTED = "Greensleeves"
+REVIEW_HEADING = "REVIEW"
 
 running_event = threading.Event()
 
@@ -157,14 +160,42 @@ def make_runner():
     return runner
 
 
-def start_server(tmp_path: Path) -> str:
+def start_server(tmp_path: Path) -> tuple[str, Config]:
     cfg = Config(whisper_url="http://localhost:9000/v1", output_dir=tmp_path / "out",
                  work_dir=tmp_path / "work", bind_host="127.0.0.1", port=8811)
     app = create_app(cfg, runner=make_runner())
     server = uvicorn.Server(uvicorn.Config(app, host=cfg.bind_host, port=cfg.port,
                                             log_level="warning"))
     threading.Thread(target=server.run, daemon=True).start()
-    return f"http://{cfg.bind_host}:{cfg.port}"
+    return f"http://{cfg.bind_host}:{cfg.port}", cfg
+
+
+def install_media(cfg: Config) -> bool:
+    """Give the selected song a playable folder, so the shot shows the player.
+
+    A flat fill rather than a test pattern: the point is the transport and the
+    lyric tracking underneath it, not the picture. Skipped without ffmpeg.
+    """
+    if shutil.which("ffmpeg") is None:
+        print("ffmpeg not found; screenshotting without the player")
+        return False
+    artist, title = next((a, t) for t, a, _, _ in SONGS if t == SELECTED)
+    slug = f"{artist} - {title}"
+    out = cfg.output_dir / slug
+    out.mkdir(parents=True, exist_ok=True)
+    subprocess.run(
+        ["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi",
+         "-i", "color=c=0x1A1430:s=640x360:d=60", "-c:v", "libx264",
+         "-pix_fmt", "yuv420p", "-preset", "ultrafast", "-an",
+         str(out / f"{slug}.mp4")], check=True)
+    subprocess.run(
+        ["ffmpeg", "-y", "-loglevel", "error", "-f", "lavfi",
+         "-i", "anullsrc=r=44100:cl=stereo:d=60", "-b:a", "64k",
+         str(out / f"{slug}.mp3")], check=True)
+    (out / f"{slug}.txt").write_text(
+        f"#TITLE:{title}\n#ARTIST:{artist}\n#MP3:{slug}.mp3\n#VIDEO:{slug}.mp4\n"
+        f"#BPM:240\n#GAP:10000\n: 0 4 16 Alas,\nE\n", encoding="utf-8")
+    return True
 
 
 def seed_jobs(base_url: str) -> None:
@@ -187,13 +218,25 @@ def seed_jobs(base_url: str) -> None:
                 time.sleep(0.05)
 
 
-def capture(base_url: str) -> None:
+def capture(base_url: str, with_player: bool) -> None:
     with sync_playwright() as p:
         browser = p.chromium.launch()
-        page = browser.new_page(viewport={"width": 1280, "height": 900})
+        page = browser.new_page(viewport={"width": 1280, "height": 1000})
         page.goto(base_url, wait_until="networkidle")
         page.get_by_text(SELECTED, exact=True).click()
-        page.wait_for_timeout(300)
+        page.wait_for_timeout(500)
+        if with_player:
+            # paused mid-line, so the shot carries the playhead and the sung /
+            # current word colouring rather than an untouched transport
+            page.evaluate("document.getElementById('player-media').currentTime = 11.6")
+            page.wait_for_timeout(400)
+            # scrolled to the review: the player pins to the top of #main and
+            # the tracked lines sit under it, which is the panel in use
+            page.evaluate("""() => {
+              const main = document.getElementById('main');
+              main.scrollTo(0, document.getElementById('review').offsetTop - 12);
+            }""")
+            page.wait_for_timeout(400)
         OUT_IMAGE.parent.mkdir(parents=True, exist_ok=True)
         page.screenshot(path=str(OUT_IMAGE))
         browser.close()
@@ -201,10 +244,11 @@ def capture(base_url: str) -> None:
 
 def main() -> None:
     with tempfile.TemporaryDirectory() as tmp:
-        base_url = start_server(Path(tmp))
+        base_url, cfg = start_server(Path(tmp))
         time.sleep(1)
         seed_jobs(base_url)
-        capture(base_url)
+        with_player = install_media(cfg)
+        capture(base_url, with_player)
     print(f"wrote {OUT_IMAGE}")
 
 

src/songtrace/stages/package.py

@@ -51,11 +51,15 @@ def _run(ctx: Ctx) -> None:
     meta = json.loads((ctx.work / "meta.json").read_text())
     src_video = ctx.work / meta["video_file"]
     fmt, codec = ffprobe_codec(src_video)
+    # The video is background only: the song's audio is #MP3, and the two are
+    # the same recording. A player that does not mute the video plays both at
+    # once, which is heard as an echo, so the audio track is dropped here.
     if (fmt, codec) in GOOD_VIDEO:
-        shutil.copyfile(src_video, out / f"{slug}{src_video.suffix}")
+        ffmpeg("-i", str(src_video), "-c", "copy", "-an",
+               str(out / f"{slug}{src_video.suffix}"))
     else:
         ffmpeg("-i", str(src_video), "-c:v", "libx264", "-crf", "20",
-               "-preset", "medium", "-c:a", "aac", str(out / f"{slug}.mp4"))
+               "-preset", "medium", "-an", str(out / f"{slug}.mp4"))
         _set_header(txt, "VIDEO", f"{slug}.mp4")
     cover_src = ctx.inp.cover_path
     if cover_src is None and meta.get("thumbnail"):

src/songtrace/static/index.html

@@ -111,6 +111,7 @@ body {
 }
 
 #sidebar .eyebrow, #jobs .empty-note { padding: 0 18px; }
+#create-song { display: block; width: calc(100% - 36px); margin: 0 18px 20px; padding: 9px 12px; }
 
 #jobs { display: flex; flex-direction: column; gap: 1px; margin-top: 8px; }
 
@@ -320,8 +321,27 @@ button { cursor: pointer; }
 }
 #review h3 span { color: var(--text-muted); font-weight: 400; }
 
+/* sticks to the top of #main so the video stays in frame as the lyrics scroll */
+#player { position: sticky; top: 0; z-index: 2; margin: 0 0 14px; padding: 6px 0 10px; background: var(--bg); }
+#player-video {
+  display: block;
+  width: 100%;
+  max-height: 30vh;
+  background: #000;
+  border: 1px solid var(--border);
+  border-radius: 2px;
+}
+#player-video[hidden] { display: none; }
+/* the native transport renders light otherwise, against everything around it */
+#player-media { display: block; width: 100%; margin-top: 8px; color-scheme: dark; }
+
 #lyrics-review { display: flex; flex-direction: column; gap: 10px; }
 .review-line { padding: 8px 12px; background: var(--panel); border: 1px solid var(--border); }
+.review-line-active { background: var(--panel-raised); border-color: var(--aqua); }
+.review-word-sung { color: var(--text-muted); }
+.review-word-active { color: var(--aqua); font-weight: 600; }
+.review-playhead { position: absolute; top: 0; bottom: 0; width: 2px; margin-left: -1px; background: var(--aqua); }
+.review-seek { cursor: pointer; }
 .review-line-text { display: block; font-size: 0.9rem; }
 .review-line-dropped { opacity: 0.55; }
 .review-line-dropped .review-line-text { text-decoration: line-through; }
@@ -355,7 +375,7 @@ button { cursor: pointer; }
   border: 1px solid var(--border);
 }
 .output-path code { flex: 1; overflow-wrap: anywhere; font-family: var(--font-mono); font-size: 0.82rem; }
-#copy-path, .action, #cancel-rebuild {
+#copy-path, .action {
   flex: none;
   padding: 5px 12px;
   font-family: var(--font-display);
@@ -368,7 +388,7 @@ button { cursor: pointer; }
   border: 1px solid var(--control);
   border-radius: 2px;
 }
-#copy-path:hover, .action:hover, #cancel-rebuild:hover { border-color: var(--text-muted); }
+#copy-path:hover, .action:hover { border-color: var(--text-muted); }
 
 .form-actions { display: flex; align-items: center; gap: 14px; }
 .keep-note { margin: -8px 0 0; font-size: 0.78rem; color: var(--aqua); }
@@ -495,6 +515,7 @@ button { cursor: pointer; }
     <div class="kana" aria-hidden="true">カラオケ</div>
   </div>
   <div>
+    <button type="button" id="create-song" class="primary">CREATE SONG</button>
     <h2 class="eyebrow">QUEUE</h2>
     <div id="jobs"></div>
   </div>
@@ -533,7 +554,6 @@ button { cursor: pointer; }
       </div>
       <div class="form-actions">
         <button type="submit" class="primary">CREATE SONG</button>
-        <button type="button" id="cancel-rebuild" hidden>NEW SONG INSTEAD</button>
         <p id="form-lock" hidden></p>
       </div>
     </form>
@@ -545,6 +565,10 @@ button { cursor: pointer; }
     <h3 id="review-title"></h3>
     <p id="review-lineage" hidden></p>
     <p id="review-error" class="review-error" hidden></p>
+    <div id="player" hidden>
+      <video id="player-video" muted playsinline preload="metadata"></video>
+      <audio id="player-media" controls preload="metadata"></audio>
+    </div>
     <div id="lyrics-review"><p class="empty-note">no songs queued yet — nothing to review</p></div>
     <div id="output-path"></div>
     <div id="review-actions"></div>
@@ -575,7 +599,7 @@ const nowTitle = document.getElementById("now-title");
 const stageTrack = document.getElementById("stage-track");
 const reviewActions = document.getElementById("review-actions");
 const reviewLineage = document.getElementById("review-lineage");
-const cancelRebuild = document.getElementById("cancel-rebuild");
+const createSong = document.getElementById("create-song");
 const formLock = document.getElementById("form-lock");
 const formHeading = document.querySelector("#new-song h2");
 const submitButton = form.querySelector("button[type=submit]");
@@ -615,10 +639,15 @@ form.addEventListener("submit", async (e) => {
   if (!resp.ok) {
     const body = await resp.json().catch(() => ({}));
     formError.textContent = body.detail || `error: ${resp.status}`;
-    return;  // before the reset below, so a rejected submission is never retyped
+    return;
   }
   selectedId = (await resp.json()).id;
-  exitRebuild();
+  // the fields stay as submitted: the song just sent is the obvious starting
+  // point for the next one, and nothing has to be retyped to correct it. The
+  // queue's CREATE SONG button is what empties the form.
+  rebuildOf = null;
+  formDirty = false;
+  setFormMode(null, null);
   refreshJobs();
 });
 
@@ -647,7 +676,15 @@ for (const input of form.querySelectorAll('input[type="file"]')) {
 // anything of the operator's in there worth warning about first
 form.addEventListener("input", () => { formDirty = true; });
 
-cancelRebuild.addEventListener("click", exitRebuild);
+// always available, and the form is usually holding a selected song by now, so
+// this both clears it and brings it back into view from wherever playback
+// scrolled the panel to
+createSong.addEventListener("click", () => {
+  if (formDirty && !confirm("Discard your unsaved edits to this form?")) return;
+  exitRebuild();
+  document.getElementById("new-song").scrollIntoView({ block: "start", behavior: "smooth" });
+  form.elements.title.focus();
+});
 
 function jobCard(job) {
   const card = document.createElement("button");
@@ -873,6 +910,10 @@ function lineTimeline(words, notes, minPitch, span) {
   }
   box.append(wordsRow);
   if (notes.length) box.append(pianoRoll(notes, minPitch, span, t0, t1));
+  const playhead = document.createElement("div");
+  playhead.className = "review-playhead";
+  playhead.hidden = true;
+  box.append(playhead);
   return box;
 }
 
@@ -906,6 +947,21 @@ function lineNode(line, minPitch, span) {
   return row;
 }
 
+// What playback needs from a rendered line: the time range it was laid out
+// over, the word labels along it, and the bar that sweeps across it.
+function lineEntry(row, line) {
+  const laid = line.words.length ? line.words : line.notes;
+  if (!laid.length || line.dropped) return null;
+  const labels = [...row.querySelectorAll(".review-word")];
+  return {
+    row,
+    t0: laid[0].start,
+    t1: Math.max(...laid.map(x => x.end)),
+    words: line.words.map((w, i) => ({ el: labels[i], start: w.start, end: w.end })),
+    playhead: row.querySelector(".review-playhead"),
+  };
+}
+
 function buildLyricsReview(data) {
   const allNotes = data.lines.flatMap(l => l.notes);
   const pitches = allNotes.map(n => n.pitch);
@@ -914,6 +970,7 @@ function buildLyricsReview(data) {
   const span = Math.max(1, maxPitch - minPitch);
   const spans = [...data.missing_spans].sort((a, b) => a.start - b.start);
   const nodes = [];
+  const entries = [];
   let spanIdx = 0;
   for (const line of data.lines) {
     const lineStart = line.notes.length ? Math.min(...line.notes.map(n => n.start)) : null;
@@ -921,24 +978,172 @@ function buildLyricsReview(data) {
       nodes.push(missingSpanNode(spans[spanIdx]));
       spanIdx++;
     }
-    nodes.push(lineNode(line, minPitch, span));
+    const row = lineNode(line, minPitch, span);
+    const entry = lineEntry(row, line);
+    if (entry) entries.push(entry);
+    nodes.push(row);
   }
   while (spanIdx < spans.length) {
     nodes.push(missingSpanNode(spans[spanIdx]));
     spanIdx++;
   }
-  return nodes;
+  return { nodes, entries };
+}
+
+// The song folder's own media, played against the same seconds the review was
+// built from, so the highlighted line is the one you are hearing.
+//
+// Modelled on the game: #MP3 is the audio and the clock, the video is muted
+// background slaved to it. Playing the video's own audio alongside the mp3 is
+// the same recording twice, a few frames apart, which is heard as an echo.
+const player = document.getElementById("player");
+const media = document.getElementById("player-media");
+const video = document.getElementById("player-video");
+const DRIFT = 0.25;  // seconds before the picture is worth re-seeking
+
+function slaveVideo(force) {
+  if (video.hidden || !video.getAttribute("src")) return;
+  if (force || Math.abs(video.currentTime - media.currentTime) > DRIFT) {
+    video.currentTime = media.currentTime;
+  }
+}
+
+for (const event of ["seeked", "ratechange"]) {
+  media.addEventListener(event, () => {
+    video.playbackRate = media.playbackRate;
+    slaveVideo(true);
+  });
+}
+media.addEventListener("pause", () => video.pause());
+let timeline = [];
+let activeEntry = null;
+let syncFrame = null;
+
+// the first line that has not finished yet: the one being sung, or the one
+// coming up while the track is between lines
+function entryAt(t) {
+  return timeline.find(e => t <= e.t1) ?? timeline[timeline.length - 1] ?? null;
+}
+
+function paintEntry(entry, t) {
+  for (const w of entry.words) {
+    w.el.classList.toggle("review-word-sung", t >= w.end);
+    w.el.classList.toggle("review-word-active", t >= w.start && t < w.end);
+  }
+  if (!entry.playhead) return;
+  const inside = t >= entry.t0 && t <= entry.t1;
+  entry.playhead.hidden = !inside;
+  if (inside) {
+    const frac = (t - entry.t0) / Math.max(entry.t1 - entry.t0, 0.001);
+    entry.playhead.style.left = `${frac * 100}%`;
+  }
+}
+
+function clearEntry(entry) {
+  entry.row.classList.remove("review-line-active");
+  if (entry.playhead) entry.playhead.hidden = true;
+  for (const w of entry.words) {
+    w.el.classList.remove("review-word-sung", "review-word-active");
+  }
+}
+
+function syncPlayback() {
+  const t = media.currentTime;
+  const entry = entryAt(t);
+  if (entry !== activeEntry) {
+    if (activeEntry) clearEntry(activeEntry);
+    activeEntry = entry;
+    if (entry) {
+      entry.row.classList.add("review-line-active");
+      focusActive();
+    }
+  }
+  if (entry) paintEntry(entry, t);
+}
+
+// centred so there is context either side, like a scrolling lyric sheet; only
+// while running, so it does not fight a reader who has scrolled away to look
+// at something else
+function focusActive() {
+  if (!activeEntry || media.paused) return;
+  activeEntry.row.scrollIntoView({ block: "center", behavior: "smooth" });
+}
+
+function pumpPlayback() {
+  syncPlayback();
+  slaveVideo(false);
+  syncFrame = media.paused ? null : requestAnimationFrame(pumpPlayback);
+}
+
+// timeupdate fires about four times a second, too coarse for a playhead
+media.addEventListener("play", () => {
+  if (syncFrame === null) syncFrame = requestAnimationFrame(pumpPlayback);
+  if (!video.hidden && video.getAttribute("src")) video.play().catch(() => {});
+  focusActive();  // the line may already be active from a seek while paused
+});
+for (const event of ["pause", "seeked", "loadedmetadata"]) {
+  media.addEventListener(event, syncPlayback);
+}
+
+function bindTimeline(entries) {
+  if (activeEntry) clearEntry(activeEntry);
+  activeEntry = null;
+  timeline = entries;
+  for (const entry of entries) {
+    entry.row.classList.add("review-seek");
+    entry.row.title = "click to play from here";
+    entry.row.addEventListener("click", (e) => {
+      const word = entry.words.find(w => w.el === e.target);
+      media.currentTime = word ? word.start : entry.t0;
+      if (media.paused) syncPlayback();
+    });
+  }
+}
+
+// Returns true when the folder gave us something to play. #MP3 is required —
+// it is the audio the game uses, so it is the one the review has to track.
+function setPlayerSource(job, available) {
+  const src = available?.audio ? `/api/jobs/${job.id}/media/audio` : "";
+  player.hidden = !src;
+  if (media.dataset.src === src) return !!src;
+  media.dataset.src = src;
+  if (src) media.src = src;
+  else { media.removeAttribute("src"); media.load(); }
+
+  video.hidden = !(src && available.video);
+  video.poster = available?.cover ? `/api/jobs/${job.id}/media/cover` : "";
+  if (!video.hidden) video.src = `/api/jobs/${job.id}/media/video`;
+  else { video.removeAttribute("src"); video.load(); }
+  return !!src;
+}
+
+function resetPlayer() {
+  // the 3s poll reaches here for every running job; reloading a cleared
+  // element each time would be pure churn
+  if (player.hidden && !timeline.length) return;
+  bindTimeline([]);
+  player.hidden = true;
+  media.pause();
+  video.pause();
+  media.dataset.src = "";
+  for (const el of [media, video]) {
+    el.removeAttribute("src");
+    el.removeAttribute("poster");
+    el.load();
+  }
 }
 
 async function renderLyricsReview(job) {
   const box = document.getElementById("lyrics-review");
   if (job.status === "failed") {
     delete box.dataset.jobId;
+    resetPlayer();
     box.replaceChildren(emptyNote("job failed — nothing to review"));
     return;
   }
   if (job.status !== "done") {
     delete box.dataset.jobId;
+    resetPlayer();
     box.replaceChildren(emptyNote("waiting for this song to finish…"));
     return;
   }
@@ -946,6 +1151,7 @@ async function renderLyricsReview(job) {
   const resp = await fetch(`/api/jobs/${job.id}/review`);
   if (job.id !== selectedId) return;
   if (!resp.ok) {
+    resetPlayer();
     box.replaceChildren(emptyNote("no review data available"));
     return;
   }
@@ -953,10 +1159,17 @@ async function renderLyricsReview(job) {
   if (job.id !== selectedId) return;
   box.dataset.jobId = job.id;
   if (!data.lines.length) {
+    resetPlayer();
     box.replaceChildren(emptyNote("no lyrics to review"));
     return;
   }
-  box.replaceChildren(...buildLyricsReview(data));
+  const { nodes, entries } = buildLyricsReview(data);
+  box.replaceChildren(...nodes);
+  // with nothing to play, a highlighted line and a click-to-seek cursor would
+  // both be promising something the song folder cannot deliver
+  const playable = setPlayerSource(job, data.media);
+  bindTimeline(playable ? entries : []);
+  if (playable) syncPlayback();
 }
 
 async function copyPath(path, button) {
@@ -1020,6 +1233,7 @@ function clearDetail() {
   reviewTitle.replaceChildren();
   reviewError.hidden = true;
   reviewError.textContent = "";
+  resetPlayer();
   document.getElementById("warnings").replaceChildren(
     emptyNote("no songs queued yet — nothing to review"));
   outputPathDiv.replaceChildren();
@@ -1043,7 +1257,6 @@ function setSubmitLock(status) {
 function setFormMode(inp, status) {
   formHeading.textContent = rebuildOf ? "REBUILD" : "NEW SONG";
   submitButton.textContent = rebuildOf ? "REBUILD SONG" : "CREATE SONG";
-  cancelRebuild.hidden = !rebuildOf;
   setSubmitLock(status);
   for (const [field, key] of [["video", "video_filename"], ["cover", "cover_filename"]]) {
     const note = form.querySelector(`[data-keep="${field}"]`);
@@ -1052,9 +1265,9 @@ function setFormMode(inp, status) {
     note.hidden = !kept;
   }
   // the advanced fields carry into a rebuild whether or not the section is open,
-  // so a job that has any must not hide them behind a closed summary
-  form.querySelector("details").open = !!inp && (!!inp.cover_filename ||
-    ["genre", "year", "language", "bpm"].some(n => form.elements[n].value));
+  // so anything actually set must not sit hidden behind a closed summary
+  form.querySelector("details").open = !!inp?.cover_filename ||
+    ["genre", "year", "language", "bpm"].some(n => form.elements[n].value);
 }
 
 function exitRebuild() {

src/songtrace/web.py

@@ -19,9 +19,12 @@ from .config import Config
 from .lyrics import normalize_lyrics
 from .pipeline import (STAGE_ORDER, PipelineError, SongInput, read_warnings,
                        run_pipeline, song_slug)
+from .ultrastar import get_header, parse_ultrastar
 
 STATIC = Path(__file__).parent / "static"
 STAGE_LINE = re.compile(r"^\[(\w+)\] (?:running|cached, skipping)")
+# the song folder names its own media, exactly as the game reads it
+MEDIA_HEADERS = {"audio": "MP3", "video": "VIDEO", "cover": "COVER"}
 
 
 @dataclass
@@ -376,6 +379,35 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
         shutil.rmtree(cfg.work_dir / slug, ignore_errors=True)
         shutil.rmtree(cfg.output_dir / slug, ignore_errors=True)
 
+    def _resolve_media(job: Job, header: str) -> Path | None:
+        """The file a song folder's #MP3/#VIDEO/#COVER header points at."""
+        slug = song_slug(job.inp)
+        out = (cfg.output_dir / slug).resolve()
+        txt = out / f"{slug}.txt"
+        if not txt.exists():
+            return None
+        name = get_header(parse_ultrastar(txt.read_text(encoding="utf-8")), header)
+        if not name:
+            return None
+        # a header names a file in the song folder and nothing else
+        path = (out / name).resolve()
+        if path.parent != out or not path.is_file():
+            return None
+        return path
+
+    @app.get("/api/jobs/{job_id}/media/{kind}")
+    def get_media(job_id: str, kind: str):
+        header = MEDIA_HEADERS.get(kind)
+        if header is None:
+            raise HTTPException(404)
+        job = _finished_job(job_id)
+        if job.status != "done":
+            raise HTTPException(404)
+        path = _resolve_media(job, header)
+        if path is None:
+            raise HTTPException(404)
+        return FileResponse(path)
+
     @app.get("/api/jobs/{job_id}/review")
     def get_review(job_id: str):
         job = _finished_job(job_id)
@@ -406,7 +438,9 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
         lines = [{"index": i, "text": " ".join(words), "dropped": i in dropped,
                   "notes": by_line.get(i, []), "words": words_by_line.get(i, [])}
                  for i, words in enumerate(ref.lines)]
-        return {"lines": lines, "missing_spans": missing_spans}
+        media = {kind: _resolve_media(job, header) is not None
+                 for kind, header in MEDIA_HEADERS.items()}
+        return {"lines": lines, "missing_spans": missing_spans, "media": media}
 
     return app
 

tests/test_package.py

@@ -49,19 +49,37 @@ def make_pipeline_inputs(tmp_path, cover=None):
     return inp, cfg
 
 
-def test_package_copies_compatible_video(tmp_path, monkeypatch):
-    monkeypatch.setattr(package, "ffmpeg", lambda *a: Path(a[-1]).write_bytes(b"m"))
+def test_package_remuxes_compatible_video_without_reencoding(tmp_path, monkeypatch):
+    calls = []
+    monkeypatch.setattr(package, "ffmpeg",
+                        lambda *a: (calls.append(a), Path(a[-1]).write_bytes(b"m")))
     monkeypatch.setattr(package, "ffprobe_codec", lambda p: ("webm", "vp9"))
     ctx = make_ctx(tmp_path, cover=make_cover(tmp_path))
     package.package_stage().run(ctx)
     out = tmp_path / "out" / "A - T"
     assert (out / "A - T.txt").exists()
     assert (out / "A - T.mp3").exists()
-    assert (out / "A - T.webm").read_bytes() == b"v"
+    assert (out / "A - T.webm").exists()
+    video_call = next(c for c in calls if c[-1].endswith("A - T.webm"))
+    assert "-c" in video_call and "copy" in video_call, "the video was re-encoded"
     img = Image.open(out / "A - T.jpg")
     assert img.width == img.height
 
 
+@pytest.mark.parametrize("probe,name", [(("webm", "vp9"), "A - T.webm"),
+                                        (("avi", "mpeg4"), "A - T.mp4")])
+def test_package_strips_the_videos_audio_track(tmp_path, monkeypatch, probe, name):
+    """#MP3 is the song's audio; a video that keeps its own plays as an echo."""
+    calls = []
+    monkeypatch.setattr(package, "ffmpeg",
+                        lambda *a: (calls.append(a), Path(a[-1]).write_bytes(b"m")))
+    monkeypatch.setattr(package, "ffprobe_codec", lambda p: probe)
+    ctx = make_ctx(tmp_path, cover=make_cover(tmp_path))
+    package.package_stage().run(ctx)
+    video_call = next(c for c in calls if c[-1].endswith(name))
+    assert "-an" in video_call
+
+
 def test_package_reencodes_and_rewrites_video_header(tmp_path, monkeypatch):
     monkeypatch.setattr(package, "ffmpeg", lambda *a: Path(a[-1]).write_bytes(b"m"))
     monkeypatch.setattr(package, "ffprobe_codec", lambda p: ("avi", "mpeg4"))

tests/test_web.py

@@ -584,11 +584,19 @@ def _submit_body(html):
     return html[start:html.index("\n});\n", start)]
 
 
+def _listener_body(html, prefix):
+    start = html.index(prefix)
+    return html[start:html.index("\n});\n", start)]
+
+
 def _rebuild_flaws(html):
     """Ways rebuild mode could quietly stop working."""
     flaws, submit = [], _submit_body(html)
     if "rebuildOf ?" not in submit or "/rebuild" not in submit:
         flaws.append("the form always posts to /api/jobs, so edits create a new song")
+    if "form.reset()" in submit or "exitRebuild()" in submit:
+        flaws.append("a successful submission empties the form, so correcting "
+                     "or varying the song just sent means retyping it")
 
     actions = _fn_body(html, "renderActions")
     if not re.search(r"if \(.*dataset\.jobId === job\.id.*\) return;", actions):
@@ -609,6 +617,14 @@ def _rebuild_flaws(html):
     if "if (rebuildOf === id) exitRebuild();" not in _fn_body(html, "deleteJob"):
         flaws.append("deleting the loaded job leaves the form pointed at it")
 
+    # selecting a queue entry loads it, so the sidebar button is now the only
+    # way back to a blank form
+    create = _listener_body(html, 'createSong.addEventListener("click"')
+    if "exitRebuild()" not in create:
+        flaws.append("the queue's CREATE SONG button does not clear the form")
+    if "formDirty" not in create or "confirm(" not in create:
+        flaws.append("the queue's CREATE SONG button discards edits without asking")
+
     mode = _fn_body(html, "setFormMode")
     if 'querySelector("details").open' not in mode:
         flaws.append("prefilled genre/year/language/bpm stay hidden behind the "
@@ -656,6 +672,54 @@ def test_review_panel_resets_when_no_job_is_selected():
     assert _empty_state_flaws(_index_html()) == []
 
 
+def _playback_flaws(html):
+    """Ways the review could stop tracking the media element."""
+    flaws = []
+    source = _fn_body(html, "setPlayerSource")
+    if "media/audio" not in source:
+        flaws.append("the player is not pointed at the job's packaged audio, "
+                     "which is the file the game plays and the notes track")
+    if "media/video" not in source:
+        flaws.append("the packaged video is never shown alongside the audio")
+
+    # the mp3 and the video carry the same recording; playing both is an echo
+    if not re.search(r"<video[^>]*\bmuted\b", html):
+        flaws.append("the background video is not muted, so its own audio "
+                     "plays on top of the mp3")
+    pump = _fn_body(html, "pumpPlayback")
+    if "slaveVideo(" not in pump:
+        flaws.append("the picture is never pulled back onto the audio clock, "
+                     "so it drifts out of sync over a long song")
+
+    if "requestAnimationFrame" not in _fn_body(html, "pumpPlayback"):
+        flaws.append("sync is not driven by the frame loop, so the playhead "
+                     "steps at timeupdate's ~4Hz instead of following the audio")
+
+    if "media.paused" not in _fn_body(html, "focusActive"):
+        flaws.append("the view is scrolled even while paused, fighting a reader "
+                     "who scrolled away to look at something else")
+
+    sync = _fn_body(html, "syncPlayback")
+    if "clearEntry(" not in sync:
+        flaws.append("the line the playhead left keeps its highlight")
+    clear = _fn_body(html, "clearEntry")
+    for name in ("review-word-sung", "review-word-active", "review-line-active"):
+        if name not in clear:
+            flaws.append(f"{name} is never taken off a line the playhead left")
+
+    # every branch that gives up on rendering a review has to unload the media
+    # too, or the previous song stays loaded and playable under the new one
+    render = _fn_body(html, "renderLyricsReview")
+    if render.count("resetPlayer()") < render.count("emptyNote("):
+        flaws.append("a review the page cannot render leaves the previous "
+                     "song's media loaded in the player")
+    return flaws
+
+
+def test_review_playback_tracks_the_media_element():
+    assert _playback_flaws(_index_html()) == []
+
+
 # each mutation is a plausible regression; the checker above must reject all of them
 MUTATIONS = [
     (_focus_flaws, "queue: card moved unconditionally",
@@ -700,6 +764,34 @@ MUTATIONS = [
      'form.querySelector("details").open =', "const unusedAdvancedOpen ="),
     (_rebuild_flaws, "prefill: an in-progress job can be submitted for rebuild anyway",
      "submitButton.disabled = locked;", ""),
+    (_rebuild_flaws, "submit: a successful submission empties the form again",
+     "  rebuildOf = null;\n  formDirty = false;\n  setFormMode(null, null);\n  refreshJobs();",
+     "  exitRebuild();\n  refreshJobs();"),
+    (_rebuild_flaws, "create: the queue button no longer clears the form",
+     "  exitRebuild();\n  document.getElementById(\"new-song\")", "  document.getElementById(\"new-song\")"),
+    (_rebuild_flaws, "create: the queue button discards edits without asking",
+     '  if (formDirty && !confirm("Discard your unsaved edits to this form?")) return;\n  exitRebuild();',
+     "  exitRebuild();"),
+    (_playback_flaws, "playback: sync falls back to the timeupdate event's ~4Hz",
+     "syncFrame = media.paused ? null : requestAnimationFrame(pumpPlayback);",
+     "syncFrame = null;"),
+    (_playback_flaws, "playback: the view is scrolled even while paused",
+     "if (!activeEntry || media.paused) return;", "if (!activeEntry) return;"),
+    (_playback_flaws, "playback: the line the playhead left keeps its highlight",
+     "    if (activeEntry) clearEntry(activeEntry);\n", ""),
+    (_playback_flaws, "playback: a word stays lit after the playhead moves on",
+     'w.el.classList.remove("review-word-sung", "review-word-active");',
+     'w.el.classList.remove("review-word-sung");'),
+    (_playback_flaws, "playback: an unrenderable review leaves the old song loaded",
+     "    resetPlayer();\n", ""),
+    (_playback_flaws, "playback: the player is not pointed at the packaged audio",
+     'const src = available?.audio ? `/api/jobs/${job.id}/media/audio` : "";',
+     'const src = "";'),
+    (_playback_flaws, "playback: the background video plays its own audio too",
+     '<video id="player-video" muted playsinline',
+     '<video id="player-video" playsinline'),
+    (_playback_flaws, "playback: the picture is never re-synced to the audio",
+     "  slaveVideo(false);\n", ""),
     (_empty_state_flaws, "empty state: clearDetail no longer wired into refreshJobs",
      "if (selectedId) refreshDetail();\n  else clearDetail();",
      "if (selectedId) refreshDetail();"),
@@ -738,7 +830,7 @@ def test_rebuild_mode_reuses_the_one_form():
 
 def test_index_has_rebuild_hooks():
     html = _index_html()
-    for hook in ("review-actions", "review-lineage", "cancel-rebuild",
+    for hook in ("review-actions", "review-lineage", "create-song",
                  'data-keep="video"', 'data-keep="cover"'):
         assert hook in html, hook
 
@@ -1385,3 +1477,152 @@ def test_review_missing_when_pipeline_never_wrote_notes(tmp_path):
     client, _ = make_client(tmp_path, ok_runner)
     job_id = make_done_job(client)
     assert client.get(f"/api/jobs/{job_id}/review").status_code == 404
+
+
+# --- review playback -------------------------------------------------------
+
+MEDIA_FILES = {"A - T.mp3": b"ID3audio", "A - T.mp4": b"\0\0\0\x18ftypvideo",
+               "A - T.jpg": b"\xff\xd8cover"}
+MEDIA_HEADERS = [("TITLE", "T"), ("ARTIST", "A"), ("MP3", "A - T.mp3"),
+                 ("COVER", "A - T.jpg"), ("VIDEO", "A - T.mp4")]
+
+
+def media_runner_factory(files=None, headers=None, write_txt=True):
+    """A run that leaves behind a song folder, the way package does."""
+    def runner(inp, cfg, log=print, force_from=None, stages=None):
+        slug = song_slug(inp)
+        out = cfg.output_dir / slug
+        out.mkdir(parents=True, exist_ok=True)
+        for name, body in (MEDIA_FILES if files is None else files).items():
+            (out / name).write_bytes(body)
+        if write_txt:
+            rows = [f"#{k}:{v}" for k, v in
+                    (MEDIA_HEADERS if headers is None else headers)]
+            (out / f"{slug}.txt").write_text("\n".join(rows) + "\n: 0 4 0 la\nE\n",
+                                             encoding="utf-8")
+        return PipelineResult(output_dir=out)
+    return runner
+
+
+def test_media_serves_the_files_the_song_txt_names(tmp_path):
+    client, _ = make_client(tmp_path, media_runner_factory())
+    job_id = make_done_job(client)
+    for kind, name in (("audio", "A - T.mp3"), ("video", "A - T.mp4"),
+                       ("cover", "A - T.jpg")):
+        resp = client.get(f"/api/jobs/{job_id}/media/{kind}")
+        assert resp.status_code == 200, kind
+        assert resp.content == MEDIA_FILES[name], kind
+
+
+def test_media_serves_whatever_the_header_was_rewritten_to(tmp_path):
+    """package transcodes to .mp4 and rewrites #VIDEO; the extension is not fixed."""
+    files = {**MEDIA_FILES, "A - T.webm": b"transcoded"}
+    headers = [(k, "A - T.webm" if k == "VIDEO" else v) for k, v in MEDIA_HEADERS]
+    client, _ = make_client(tmp_path, media_runner_factory(files, headers))
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/video").content == b"transcoded"
+
+
+def test_media_supports_range_requests(tmp_path):
+    """Without 206 the browser cannot seek, which is the point of the player."""
+    client, _ = make_client(tmp_path, media_runner_factory())
+    job_id = make_done_job(client)
+    resp = client.get(f"/api/jobs/{job_id}/media/video", headers={"Range": "bytes=0-3"})
+    assert resp.status_code == 206
+    assert resp.headers["content-range"] == f"bytes 0-3/{len(MEDIA_FILES['A - T.mp4'])}"
+    assert resp.content == MEDIA_FILES["A - T.mp4"][:4]
+
+
+def test_media_of_an_unknown_kind_is_404(tmp_path):
+    client, _ = make_client(tmp_path, media_runner_factory())
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/instrumental").status_code == 404
+
+
+def test_media_refuses_a_header_pointing_outside_the_song_folder(tmp_path):
+    secret = tmp_path / "secret.txt"
+    secret.write_text("not yours")
+    headers = [(k, "../../secret.txt" if k == "MP3" else v) for k, v in MEDIA_HEADERS]
+    client, _ = make_client(tmp_path, media_runner_factory(headers=headers))
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 404
+
+
+def test_media_refuses_an_absolute_header_path(tmp_path):
+    secret = tmp_path / "secret.txt"
+    secret.write_text("not yours")
+    headers = [(k, str(secret) if k == "MP3" else v) for k, v in MEDIA_HEADERS]
+    client, _ = make_client(tmp_path, media_runner_factory(headers=headers))
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 404
+
+
+def test_media_404s_when_the_named_file_is_missing(tmp_path):
+    client, _ = make_client(tmp_path, media_runner_factory(files={}))
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 404
+
+
+def test_media_404s_when_the_header_was_dropped(tmp_path):
+    """package strips #COVER when there is no usable cover image."""
+    headers = [(k, v) for k, v in MEDIA_HEADERS if k != "COVER"]
+    client, _ = make_client(tmp_path, media_runner_factory(headers=headers))
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/cover").status_code == 404
+    assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 200
+
+
+def test_media_404s_when_there_is_no_song_txt(tmp_path):
+    client, _ = make_client(tmp_path, media_runner_factory(write_txt=False))
+    job_id = make_done_job(client)
+    assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 404
+
+
+def test_media_of_a_failed_job_is_404(tmp_path):
+    def bad_runner(*a, **kw):
+        raise PipelineError("boom")
+    client, _ = make_client(tmp_path, bad_runner)
+    job_id = client.post("/api/jobs", data={
+        "title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"}).json()["id"]
+    wait_status(client, job_id, "failed")
+    assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 404
+
+
+def test_media_of_a_running_job_is_409(tmp_path):
+    release = threading.Event()
+
+    def blocking(inp, cfg, log=print, force_from=None, stages=None):
+        release.wait(5)
+        return PipelineResult(output_dir=cfg.output_dir / "X")
+
+    client, _ = make_client(tmp_path, blocking)
+    job_id = client.post("/api/jobs", data={
+        "title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"}).json()["id"]
+    try:
+        wait_status(client, job_id, "running")
+        assert client.get(f"/api/jobs/{job_id}/media/audio").status_code == 409
+    finally:
+        release.set()
+    wait_status(client, job_id, "done")
+
+
+def test_media_of_an_unknown_job_is_404(tmp_path):
+    client, _ = make_client(tmp_path, media_runner_factory())
+    assert client.get("/api/jobs/nope/media/audio").status_code == 404
+
+
+def test_review_reports_which_media_the_player_can_load(tmp_path):
+    notes = [{"start": 1.0, "duration": 0.4, "pitch": 0, "text": "hi",
+              "low_confidence": False, "line_index": 0}]
+    review = review_runner_factory(notes)
+    media = media_runner_factory(headers=[(k, v) for k, v in MEDIA_HEADERS
+                                          if k != "COVER"])
+
+    def runner(inp, cfg, **kw):
+        media(inp, cfg, **kw)
+        return review(inp, cfg, **kw)
+
+    client, _ = make_client(tmp_path, runner)
+    job_id = make_done_job(client, lyrics="hi")
+    body = client.get(f"/api/jobs/{job_id}/review").json()
+    assert body["media"] == {"audio": True, "video": True, "cover": False}