Merge pull request 'Webui: open video source, restructure lyrics form, and add a visual per-line review' (#4) from worktree-webui-lyrics-review into main

Reviewed-on: #4
This commit is contained in:
2026-07-28 01:42:56 -07:00
12 changed files with 743 additions and 96 deletions
Binary file not shown.

Before

Width:  |  Height:  |  Size: 57 KiB

After

Width:  |  Height:  |  Size: 77 KiB

+97 -5
View File
@@ -6,6 +6,7 @@ with a fixed set of jobs covering every status, then screenshots it.
One-time setup: pip install -e '.[dev]' && playwright install chromium
Run: python scripts/screenshot_webui.py
"""
import json
import tempfile
import threading
import time
@@ -16,12 +17,36 @@ import uvicorn
from playwright.sync_api import sync_playwright
from songtrace.config import Config
from songtrace.pipeline import PipelineError, PipelineResult
from songtrace.pipeline import PipelineError, PipelineResult, song_slug
from songtrace.web import create_app
REPO_ROOT = Path(__file__).resolve().parent.parent
OUT_IMAGE = REPO_ROOT / "docs" / "images" / "webui.png"
STAGES = ["acquire", "separate", "transcribe", "align", "pitch", "notes", "grid", "package"]
DEFAULT_LYRICS = "[verse]\n...\n[chorus]\n...\n"
# public domain (16th century English folk song, author unknown)
GREENSLEEVES_LYRICS = """\
Alas, my love, you do me wrong
To cast me off discourteously
For I have loved you well and long
Delighting in your company
Greensleeves was all my joy
Greensleeves was my delight
Greensleeves was my heart of gold
And who but my lady Greensleeves
I have been ready at your hand
To grant whatever you would crave
I have both waged life and land
Your love and good will for to have
Greensleeves was all my joy
Greensleeves was my delight
Greensleeves was my heart of gold
And who but my lady Greensleeves
"""
# order matters: the app has one worker thread, so each job runs in submission
# order. "running" must be submitted after the done/failed ones complete and
@@ -33,12 +58,58 @@ SONGS = [
("Pink Moon", "Nick Drake", "done",
{"warnings": ["2 low-confidence notes near 0:38–0:41",
"1 low-confidence note near 1:22–1:24"]}),
("Landslide", "Fleetwood Mac", "done",
{"warnings": ["1 low-confidence note near 0:52–0:55"]}),
("Greensleeves", "Traditional", "done",
{"lyrics": GREENSLEEVES_LYRICS,
"warnings": ["1 low-confidence note near 0:45–0:47",
"1 line dropped (not sung) near \"for i have loved\"",
"missing lyrics near 2:10–2:18"],
"review": {
"lines": [
# Pitches are the real Greensleeves melody (E natural minor, as
# arranged by Gilbert DeBenedetti, "Level 4 - Flowing"), converted
# to UltraStar semitones (0 = C4 = MIDI 60): E5=16 F#5=18 G5=19
# A5=21 B4=11 D5=14.
{"index": 0, "text": "Alas, my love, you do me wrong",
"dropped": False,
"notes": [{"start": 10.0, "end": 10.4, "pitch": 16, "low_confidence": False},
{"start": 10.4, "end": 10.6, "pitch": 18, "low_confidence": False},
{"start": 10.6, "end": 11.2, "pitch": 19, "low_confidence": False},
{"start": 11.2, "end": 11.5, "pitch": 21, "low_confidence": False},
{"start": 11.5, "end": 11.7, "pitch": 19, "low_confidence": False},
{"start": 11.7, "end": 11.9, "pitch": 18, "low_confidence": False},
{"start": 11.9, "end": 12.9, "pitch": 14, "low_confidence": False}],
"words": [{"text": "Alas,", "start": 10.0, "end": 10.4},
{"text": "my", "start": 10.4, "end": 10.6},
{"text": "love,", "start": 10.6, "end": 11.2},
{"text": "you", "start": 11.2, "end": 11.5},
{"text": "do", "start": 11.5, "end": 11.7},
{"text": "me", "start": 11.7, "end": 11.9},
{"text": "wrong", "start": 11.9, "end": 12.9}]},
{"index": 1, "text": "To cast me off discourteously",
"dropped": False,
"notes": [{"start": 44.0, "end": 44.3, "pitch": 11, "low_confidence": False},
{"start": 44.3, "end": 44.6, "pitch": 16, "low_confidence": False},
{"start": 44.6, "end": 44.8, "pitch": 18, "low_confidence": False},
{"start": 44.8, "end": 45.0, "pitch": 19, "low_confidence": False},
{"start": 45.0, "end": 45.4, "pitch": 18, "low_confidence": True}],
"words": [{"text": "To", "start": 44.0, "end": 44.3},
{"text": "cast", "start": 44.3, "end": 44.6},
{"text": "me", "start": 44.6, "end": 44.8},
{"text": "off", "start": 44.8, "end": 45.0},
{"text": "discourteously", "start": 45.0, "end": 45.4}]},
{"index": 2, "text": "For I have loved you well and long",
"dropped": True, "notes": [], "words": []},
],
"missing_spans": [{"start": 130.0, "end": 138.0}],
}}),
("Such Great Heights", "The Postal Service", "failed",
{"error": "yt-dlp: unable to download video data: HTTP Error 403: Forbidden"}),
("Time Escaping", "Big Thief", "running", {}),
("Vampire in the Corner", "Magdalena Bay", "queued", {}),
]
SELECTED = "Pink Moon"
SELECTED = "Greensleeves"
running_event = threading.Event()
@@ -51,6 +122,27 @@ def make_runner():
if kind == "done":
for s in STAGES:
log(f"[{s}] running")
if "review" in extra:
work = cfg.work_dir / song_slug(inp)
work.mkdir(parents=True, exist_ok=True)
notes = [
{**n, "duration": n["end"] - n["start"], "text": "~",
"line_index": line["index"]}
for line in extra["review"]["lines"] for n in line["notes"]
]
(work / "notes_sec.json").write_text(json.dumps({
"notes": notes, "breaks": [], "warnings": extra["warnings"]}))
(work / "align_meta.json").write_text(json.dumps({
"dropped_lines": [line["index"] for line in extra["review"]["lines"]
if line["dropped"]],
"missing_spans": [[s["start"], s["end"]]
for s in extra["review"]["missing_spans"]],
}))
(work / "alignment.json").write_text(json.dumps([
{"word": w["text"], "start": w["start"], "end": w["end"],
"score": 0.9, "line_index": line["index"]}
for line in extra["review"]["lines"] for w in line.get("words", [])
]))
out = cfg.output_dir / f"{inp.artist} - {inp.title}"
out.mkdir(parents=True, exist_ok=True)
return PipelineResult(output_dir=out, warnings=extra["warnings"])
@@ -76,11 +168,11 @@ def start_server(tmp_path: Path) -> str:
def seed_jobs(base_url: str) -> None:
lyrics = "[verse]\n...\n[chorus]\n...\n"
with httpx.Client(timeout=10) as client:
for title, artist, kind, _ in SONGS:
for title, artist, kind, extra in SONGS:
resp = client.post(f"{base_url}/api/jobs", data={
"title": title, "artist": artist, "lyrics": lyrics,
"title": title, "artist": artist,
"lyrics": extra.get("lyrics", DEFAULT_LYRICS),
"youtube_url": "https://youtu.be/00000000000"})
resp.raise_for_status()
job_id = resp.json()["id"]
+40 -9
View File
@@ -23,6 +23,7 @@ class TimedNote:
pitch: int # UltraStar semitone (0 = C4 = MIDI 60)
text: str # display text incl. leading space / "~"
low_confidence: bool
line_index: int
def hz_to_midi_float(hz: float) -> float:
@@ -78,21 +79,31 @@ def syllabify(word: str, language: str | None) -> list[str]:
return parts or [word]
def build_notes(rows: list[dict], times, f0, conf, language: str | None
def build_notes(rows: list[dict], times, f0, conf, language: str | None,
dropped_lines=(), missing_spans=(), ref_lines=None
) -> tuple[list[TimedNote], list[float], list[str]]:
notes: list[TimedNote] = []
breaks: list[float] = []
prev_line, prev_end = None, 0.0
prev_line, prev_end, last_seen_line = None, 0.0, None
for row in rows:
if prev_line is not None and row["line_index"] > prev_line:
# last_seen_line tracks every row (skipped or not) so a run of
# skipped rows can't trigger a duplicate break on the next row of
# the same still-unseen line; prev_line/prev_end only advance on
# rows that actually emit notes, so a skipped row can't corrupt
# line_initial or break timing for its own line's later words.
if last_seen_line is not None and row["line_index"] > last_seen_line:
breaks.append((prev_end + row["start"]) / 2)
last_seen_line = row["line_index"]
line_initial = row["line_index"] != prev_line
if any(a <= row["start"] < b for a, b in missing_spans):
continue
prev_line, prev_end = row["line_index"], row["end"]
segs = segment_span(times, f0, conf, row["start"], row["end"])
if not segs:
pitch = notes[-1].pitch if notes else 0
notes.append(_note(row["start"], row["end"] - row["start"], pitch,
_text(row["word"], 0, line_initial), True))
_text(row["word"], 0, line_initial), True,
row["line_index"]))
continue
low = row["score"] < LOW_SCORE
sylls = syllabify(row["word"], language)
@@ -100,14 +111,16 @@ def build_notes(rows: list[dict], times, f0, conf, language: str | None
segs = _subdivide(segs, len(sylls))
for i, (s, e, midi) in enumerate(segs):
text = _text(sylls[i], i, line_initial) if i < len(sylls) else "~"
notes.append(_note(s, e - s, round(midi) - 60, text, low))
notes.append(_note(s, e - s, round(midi) - 60, text, low,
row["line_index"]))
_deoverlap(notes)
return notes, breaks, _warnings(notes)
return notes, breaks, _warnings(notes, dropped_lines, missing_spans, ref_lines or [])
def _note(start: float, duration: float, pitch: int, text: str,
low: bool) -> TimedNote:
return TimedNote(float(start), float(duration), int(pitch), text, low)
low: bool, line_index: int) -> TimedNote:
return TimedNote(float(start), float(duration), int(pitch), text, low,
int(line_index))
def _deoverlap(notes: list[TimedNote]) -> None:
@@ -153,7 +166,8 @@ def _fmt(t: float) -> str:
return f"{int(t // 60)}:{int(t % 60):02d}"
def _warnings(notes: list[TimedNote]) -> list[str]:
def _warnings(notes: list[TimedNote], dropped_lines=(), missing_spans=(),
ref_lines=()) -> list[str]:
warnings, run = [], []
for n in list(notes) + [None]:
if n is not None and n.low_confidence:
@@ -164,4 +178,21 @@ def _warnings(notes: list[TimedNote]) -> list[str]:
f"{len(run)} low-confidence note{'' if len(run) == 1 else 's'} "
f"near {_fmt(first.start)}–{_fmt(last.start + last.duration)}")
run = []
for group in _consecutive_runs(dropped_lines):
n = len(group)
text = " ".join(ref_lines[group[0]]) if group[0] < len(ref_lines) else ""
warnings.append(
f"{n} line{'' if n == 1 else 's'} dropped (not sung) near \"{text}\"")
for a, b in missing_spans:
warnings.append(f"missing lyrics near {_fmt(a)}–{_fmt(b)}")
return warnings
def _consecutive_runs(indices) -> list[list[int]]:
runs = []
for i in sorted(indices):
if runs and i == runs[-1][-1] + 1:
runs[-1].append(i)
else:
runs.append([i])
return runs
+15 -10
View File
@@ -1,9 +1,9 @@
import json
from pathlib import Path
from ..lyrics import RefLyrics, alignment_token, normalize_lyrics
from ..lyrics import alignment_token, normalize_lyrics
from ..pipeline import Ctx, PipelineError, Stage
from .transcribe import load_transcript, lyrics_similarity, vocal_sections
from .transcribe import classify_lyrics_match, load_transcript, lyrics_similarity, vocal_sections
SIMILARITY_THRESHOLD = 0.4
@@ -42,16 +42,15 @@ def force_align_section(vocals_path: Path, start: float, end: float,
sum(t.score for t in s) / len(s)) for s in spans]
def _assign_lines_to_sections(ref: RefLyrics, sections) -> list[list[int]]:
"""Split lyric lines across vocal sections proportionally to section duration."""
def _assign_lines_to_sections(line_indices: list[int], sections) -> list[list[int]]:
"""Split matched lyric line indices across vocal sections proportionally to section duration."""
total = sum(b - a for a, b in sections) or 1.0
n_lines = len(ref.lines)
n = len(line_indices)
groups, elapsed, start = [], 0.0, 0
for i, (a, b) in enumerate(sections):
elapsed += b - a
stop = n_lines if i == len(sections) - 1 else min(
n_lines, round(n_lines * elapsed / total))
groups.append(list(range(start, max(stop, start))))
stop = n if i == len(sections) - 1 else min(n, round(n * elapsed / total))
groups.append(line_indices[start:max(stop, start)])
start = max(stop, start)
return groups
@@ -99,7 +98,7 @@ def _align_section(vocals: Path, sec: tuple[float, float], words: list[tuple[str
def _run(ctx: Ctx) -> None:
ref = normalize_lyrics(ctx.inp.lyrics_text)
segments = load_transcript(ctx)
segments = sorted(load_transcript(ctx), key=lambda s: s.start)
sim = lyrics_similarity(segments, ref)
if sim < SIMILARITY_THRESHOLD:
raise PipelineError(
@@ -107,15 +106,21 @@ def _run(ctx: Ctx) -> None:
"wrong lyrics or wrong recording?")
# normalised so an inverted span can never collapse a section onto one instant
sections = [(min(a, b), max(a, b)) for a, b in vocal_sections(segments)]
dropped_lines, missing_spans = classify_lyrics_match(segments, ref)
matched = [i for i in range(len(ref.lines)) if i not in set(dropped_lines)]
vocals = ctx.work / "vocals.wav"
rows = []
for sec, line_idxs in zip(sections, _assign_lines_to_sections(ref, sections)):
for sec, line_idxs in zip(sections, _assign_lines_to_sections(matched, sections)):
words = [(w, li) for li in line_idxs for w in ref.lines[li]]
if words:
rows.extend(_align_section(vocals, sec, words, ctx.inp.language,
ctx.cfg.device))
_enforce_monotonic(rows)
(ctx.work / "alignment.json").write_text(json.dumps(rows))
(ctx.work / "align_meta.json").write_text(json.dumps({
"dropped_lines": dropped_lines,
"missing_spans": [list(s) for s in missing_spans],
}))
def align_stage() -> Stage:
+9 -1
View File
@@ -3,6 +3,7 @@ from dataclasses import asdict
import numpy as np
from ..lyrics import normalize_lyrics
from ..notes import build_notes
from ..pipeline import Ctx, Stage
@@ -10,8 +11,15 @@ from ..pipeline import Ctx, Stage
def _run(ctx: Ctx) -> None:
rows = json.loads((ctx.work / "alignment.json").read_text())
data = np.load(ctx.work / "pitch.npz")
meta_path = ctx.work / "align_meta.json"
meta = json.loads(meta_path.read_text()) if meta_path.exists() else {}
dropped_lines = meta.get("dropped_lines", [])
missing_spans = [tuple(s) for s in meta.get("missing_spans", [])]
ref_lines = normalize_lyrics(ctx.inp.lyrics_text).lines
notes, breaks, warnings = build_notes(
rows, data["times"], data["f0"], data["conf"], ctx.inp.language)
rows, data["times"], data["f0"], data["conf"], ctx.inp.language,
dropped_lines=dropped_lines, missing_spans=missing_spans,
ref_lines=ref_lines)
(ctx.work / "notes_sec.json").write_text(json.dumps({
"notes": [asdict(n) for n in notes],
"breaks": breaks,
+60 -3
View File
@@ -20,10 +20,67 @@ def _tokens(text: str) -> list[str]:
return [t for w in text.split() if (t := alignment_token(w))]
def match_lyrics(segments, ref: RefLyrics):
heard, seg_of = [], []
for si, s in enumerate(segments):
for t in _tokens(s.text):
heard.append(t)
seg_of.append(si)
pairs = [(t, li) for w, li in zip(ref.words, ref.word_line_index)
if (t := alignment_token(w))]
expected = [t for t, _ in pairs]
line_of = [li for _, li in pairs]
matcher = difflib.SequenceMatcher(a=heard, b=expected)
return matcher, seg_of, line_of
def lyrics_similarity(segments, ref: RefLyrics) -> float:
heard = [t for s in segments for t in _tokens(s.text)]
expected = [t for w in ref.words if (t := alignment_token(w))]
return difflib.SequenceMatcher(a=heard, b=expected).ratio()
matcher, _, _ = match_lyrics(segments, ref)
return matcher.ratio()
def classify_lyrics_match(segments, ref: RefLyrics, pad: float = 1.0
) -> tuple[list[int], list[tuple[float, float]]]:
"""Split reference lines into matched vs. dropped, and flag sung spans no
reference line explains.
dropped_lines: reference line indices covered only by 'insert' opcodes
(present in the lyrics, never heard). missing_spans: the padded, merged
bounds of the heard segments behind 'delete' opcodes (heard, but matching
no reference word) — scoped tightly to the unmatched audio itself, not
the whole vocal section it falls in, so matched lines sharing a section
with an ad-lib keep their notes.
"""
matcher, seg_of, line_of = match_lyrics(segments, ref)
covered = [False] * len(line_of)
for tag, _, _, bj1, bj2 in matcher.get_opcodes():
if tag in ("equal", "replace"):
for j in range(bj1, bj2):
covered[j] = True
by_line = {}
for j, li in enumerate(line_of):
by_line.setdefault(li, []).append(covered[j])
dropped_lines = sorted(li for li, cov in by_line.items() if not any(cov))
spans = []
for tag, ai1, ai2, _, _ in matcher.get_opcodes():
if tag != "delete":
continue
sis = [seg_of[i] for i in range(ai1, ai2)]
start = max(0.0, min(segments[si].start for si in sis) - pad)
end = max(segments[si].end for si in sis) + pad
spans.append((start, end))
return dropped_lines, _merge_spans(spans)
def _merge_spans(spans: list[tuple[float, float]]) -> list[tuple[float, float]]:
merged: list[list[float]] = []
for a, b in sorted(spans):
if merged and a <= merged[-1][1]:
merged[-1][1] = max(merged[-1][1], b)
else:
merged.append([a, b])
return [tuple(s) for s in merged]
def _run(ctx: Ctx) -> None:
+189 -64
View File
@@ -194,6 +194,9 @@ body {
}
#song-form { display: grid; gap: 18px; }
.song-form-grid { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; }
.song-form-main { display: grid; gap: 18px; align-content: start; }
.song-form-lyrics textarea { height: 100%; flex: 1 1 auto; }
.pair { display: grid; grid-template-columns: 1fr 1fr; gap: 18px; }
.field { display: flex; flex-direction: column; gap: 6px; }
.field > span {
@@ -220,7 +223,6 @@ input:not([type="file"]):hover { border-bottom-color: var(--text-muted); }
input:not([type="file"]):focus { border-bottom-color: var(--aqua); }
textarea {
min-height: 8.5rem;
padding: 11px 13px;
line-height: 1.6;
background: var(--panel);
@@ -229,6 +231,7 @@ textarea {
}
textarea:hover { border-color: var(--text-muted); }
textarea:focus { border-color: var(--aqua); }
.song-form-lyrics textarea { min-height: 8.5rem; }
input[type="file"] {
font-size: 0.82rem;
@@ -316,20 +319,28 @@ button { cursor: pointer; }
}
#review h3 span { color: var(--text-muted); font-weight: 400; }
#warnings { display: flex; flex-direction: column; gap: 1px; }
.warn-row {
display: grid;
grid-template-columns: auto auto 1fr;
align-items: center;
gap: 10px;
padding: 10px 12px;
#lyrics-review { display: flex; flex-direction: column; gap: 10px; }
.review-line { padding: 8px 12px; background: var(--panel); border: 1px solid var(--border); }
.review-line-text { display: block; font-size: 0.9rem; }
.review-line-dropped { opacity: 0.55; }
.review-line-dropped .review-line-text { text-decoration: line-through; }
.review-line-note { display: block; margin-top: 4px; font-size: 0.72rem; color: var(--text-muted); }
.review-line-timeline { position: relative; margin-top: 4px; }
.review-lane { position: absolute; top: 0; bottom: 0; }
.review-lane-alt { background: var(--panel-raised); }
.review-word-row { position: relative; height: 18px; }
.review-word { position: absolute; top: 0; white-space: nowrap; font-size: 0.9rem; }
.review-roll { position: relative; display: block; width: 100%; height: 20px; margin-top: 4px; }
.review-note { fill: var(--aqua); }
.review-note-low { fill: var(--warn); }
.review-missing {
margin: 0;
padding: 6px 12px;
font-size: 0.78rem;
color: var(--warn);
background: var(--panel);
border: 1px solid var(--border);
border: 1px dashed var(--border);
}
.warn-row + .warn-row { margin-top: 6px; }
.warn-icon { color: var(--warn); }
.warn-count { font-size: 0.85rem; }
.warn-range { justify-self: end; font-family: var(--font-mono); font-size: 0.82rem; color: var(--text-muted); }
.review-error { margin: 0 0 14px; font-size: 0.85rem; color: var(--fail); }
@@ -464,6 +475,11 @@ button { cursor: pointer; }
#stage-track { margin-left: 0; flex-wrap: wrap; }
}
@media (max-width: 720px) {
.song-form-grid { grid-template-columns: 1fr; }
.song-form-lyrics textarea { height: auto; }
}
@media (prefers-reduced-motion: reduce) {
*, *::before, *::after { animation: none !important; transition: none !important; }
}
@@ -486,29 +502,33 @@ button { cursor: pointer; }
<section id="new-song">
<h2>NEW SONG</h2>
<form id="song-form">
<div class="pair">
<label class="field"><span>Title</span><input name="title" required></label>
<label class="field"><span>Artist</span><input name="artist" required></label>
</div>
<label class="field"><span>Lyrics</span><textarea name="lyrics" required></textarea></label>
<details>
<summary>Advanced</summary>
<div class="stack">
<div class="song-form-grid">
<div class="song-form-main">
<div class="pair">
<label class="field"><span>Title</span><input name="title" required></label>
<label class="field"><span>Artist</span><input name="artist" required></label>
</div>
<label class="field"><span>YouTube URL</span><input name="youtube_url"></label>
<label class="field"><span>Video file</span><input type="file" name="video"></label>
<p class="keep-note" data-keep="video" hidden></p>
<div class="pair">
<label class="field"><span>Genre</span><input name="genre"></label>
<label class="field"><span>Year</span><input name="year" type="number"></label>
</div>
<div class="pair">
<label class="field"><span>Language</span><input name="language"></label>
<label class="field"><span>BPM override</span><input name="bpm" type="number" step="any"></label>
</div>
<label class="field"><span>Cover image</span><input type="file" name="cover"></label>
<p class="keep-note" data-keep="cover" hidden></p>
<details>
<summary>Advanced</summary>
<div class="stack">
<div class="pair">
<label class="field"><span>Genre</span><input name="genre"></label>
<label class="field"><span>Year</span><input name="year" type="number"></label>
</div>
<div class="pair">
<label class="field"><span>Language</span><input name="language"></label>
<label class="field"><span>BPM override</span><input name="bpm" type="number" step="any"></label>
</div>
<label class="field"><span>Cover image</span><input type="file" name="cover"></label>
<p class="keep-note" data-keep="cover" hidden></p>
</div>
</details>
</div>
</details>
<label class="field song-form-lyrics"><span>Lyrics</span><textarea name="lyrics" required></textarea></label>
</div>
<div class="form-actions">
<button type="submit" class="primary">CREATE SONG</button>
<button type="button" id="cancel-rebuild" hidden>CANCEL</button>
@@ -522,7 +542,7 @@ button { cursor: pointer; }
<h3 id="review-title"></h3>
<p id="review-lineage" hidden></p>
<p id="review-error" class="review-error" hidden></p>
<div id="warnings"><p class="empty-note">no songs queued yet — nothing to review</p></div>
<div id="lyrics-review"><p class="empty-note">no songs queued yet — nothing to review</p></div>
<div id="output-path"></div>
<div id="review-actions"></div>
<details>
@@ -561,8 +581,6 @@ let selectedId = null;
let STAGES = [];
let pollTimer = null;
const WARN_RE = /^(\d+)\s+low-confidence notes?\s+near\s+(\d+:\d{2})–(\d+:\d{2})/;
const STATE = {
queued: { glyph: "○", cls: "st-queued" },
running: { glyph: "◈", cls: "st-running" },
@@ -777,44 +795,151 @@ function emptyNote(text) {
return p;
}
function renderWarnings(job) {
const box = document.getElementById("warnings");
const SVG_NS = "http://www.w3.org/2000/svg";
function fmtTime(t) {
const s = Math.round(t);
return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, "0")}`;
}
function missingSpanNode(span) {
const p = document.createElement("p");
p.className = "review-missing";
p.textContent = `♪ unmatched singing, ${fmtTime(span.start)}–${fmtTime(span.end)}`;
return p;
}
function pianoRoll(notes, minPitch, span, t0, t1) {
const svg = document.createElementNS(SVG_NS, "svg");
svg.setAttribute("class", "review-roll");
svg.setAttribute("viewBox", "0 0 100 20");
svg.setAttribute("preserveAspectRatio", "none");
const dur = Math.max(t1 - t0, 0.001);
for (const n of notes) {
const rect = document.createElementNS(SVG_NS, "rect");
const x = ((n.start - t0) / dur) * 100;
const w = Math.max(((n.end - n.start) / dur) * 100, 0.5);
const y = 18 - ((n.pitch - minPitch) / span) * 16;
rect.setAttribute("x", x.toFixed(2));
rect.setAttribute("width", w.toFixed(2));
rect.setAttribute("y", y.toFixed(2));
rect.setAttribute("height", "2");
rect.setAttribute("class", n.low_confidence ? "review-note review-note-low" : "review-note");
svg.append(rect);
}
return svg;
}
// Lays words out along a shared time axis (like a karaoke lyric line) so
// each word's notes render directly beneath it instead of as one flat strip.
function lineTimeline(words, notes, minPitch, span) {
const t0 = words[0].start;
const t1 = words[words.length - 1].end;
const dur = Math.max(t1 - t0, 0.001);
const box = document.createElement("div");
box.className = "review-line-timeline";
words.forEach((w, i) => {
const lane = document.createElement("div");
lane.className = "review-lane" + (i % 2 ? " review-lane-alt" : "");
lane.style.left = `${((w.start - t0) / dur) * 100}%`;
lane.style.width = `${Math.max(((w.end - w.start) / dur) * 100, 0.5)}%`;
box.append(lane);
});
const wordsRow = document.createElement("div");
wordsRow.className = "review-word-row";
for (const w of words) {
const label = document.createElement("span");
label.className = "review-word";
label.style.left = `${((w.start - t0) / dur) * 100}%`;
label.textContent = w.text;
wordsRow.append(label);
}
box.append(wordsRow);
if (notes.length) box.append(pianoRoll(notes, minPitch, span, t0, t1));
return box;
}
function lineNode(line, minPitch, span) {
const row = document.createElement("div");
row.className = "review-line" + (line.dropped ? " review-line-dropped" : "");
if (line.dropped) {
const text = document.createElement("span");
text.className = "review-line-text";
text.textContent = line.text;
row.append(text);
const note = document.createElement("span");
note.className = "review-line-note";
note.textContent = "not sung — dropped";
row.append(note);
return row;
}
if (line.words.length) {
row.append(lineTimeline(line.words, line.notes, minPitch, span));
return row;
}
const text = document.createElement("span");
text.className = "review-line-text";
text.textContent = line.text;
row.append(text);
if (line.notes.length) {
const t0 = line.notes[0].start;
const t1 = Math.max(...line.notes.map(n => n.end));
row.append(pianoRoll(line.notes, minPitch, span, t0, t1));
}
return row;
}
function buildLyricsReview(data) {
const allNotes = data.lines.flatMap(l => l.notes);
const pitches = allNotes.map(n => n.pitch);
const minPitch = pitches.length ? Math.min(...pitches) : 0;
const maxPitch = pitches.length ? Math.max(...pitches) : 0;
const span = Math.max(1, maxPitch - minPitch);
const spans = [...data.missing_spans].sort((a, b) => a.start - b.start);
const nodes = [];
let spanIdx = 0;
for (const line of data.lines) {
const lineStart = line.notes.length ? Math.min(...line.notes.map(n => n.start)) : null;
while (spanIdx < spans.length && lineStart !== null && spans[spanIdx].start < lineStart) {
nodes.push(missingSpanNode(spans[spanIdx]));
spanIdx++;
}
nodes.push(lineNode(line, minPitch, span));
}
while (spanIdx < spans.length) {
nodes.push(missingSpanNode(spans[spanIdx]));
spanIdx++;
}
return nodes;
}
async function renderLyricsReview(job) {
const box = document.getElementById("lyrics-review");
if (job.status === "failed") {
delete box.dataset.jobId;
box.replaceChildren(emptyNote("job failed — nothing to review"));
return;
}
if (job.status !== "done") {
delete box.dataset.jobId;
box.replaceChildren(emptyNote("waiting for this song to finish…"));
return;
}
if (!job.warnings.length) {
box.replaceChildren(emptyNote("no low-confidence regions — nothing to review"));
if (box.dataset.jobId === job.id) return;
const resp = await fetch(`/api/jobs/${job.id}/review`);
if (job.id !== selectedId) return;
if (!resp.ok) {
box.replaceChildren(emptyNote("no review data available"));
return;
}
const rows = job.warnings.map(w => {
const m = WARN_RE.exec(w);
const row = document.createElement("div");
row.className = "warn-row";
if (m) {
row.setAttribute("data-count", m[1]);
row.setAttribute("data-start", m[2]);
row.setAttribute("data-end", m[3]);
const icon = document.createElement("span");
icon.className = "warn-icon";
icon.textContent = "⚠";
const count = document.createElement("span");
count.className = "warn-count";
count.textContent = `${m[1]} note${m[1] === "1" ? "" : "s"}`;
const range = document.createElement("span");
range.className = "warn-range";
range.textContent = `${m[2]}–${m[3]}`;
row.append(icon, count, range);
} else {
row.textContent = w;
}
return row;
});
box.replaceChildren(...rows);
const data = await resp.json();
if (job.id !== selectedId) return;
box.dataset.jobId = job.id;
if (!data.lines.length) {
box.replaceChildren(emptyNote("no lyrics to review"));
return;
}
box.replaceChildren(...buildLyricsReview(data));
}
async function copyPath(path, button) {
@@ -969,7 +1094,7 @@ async function refreshDetail() {
reviewTitle.replaceChildren(`${job.title} `, artist);
reviewError.hidden = !job.error;
reviewError.textContent = job.error ?? "";
renderWarnings(job);
renderLyricsReview(job);
renderOutputPath(job);
renderLineage(job);
renderActions(job);
+34
View File
@@ -1,3 +1,4 @@
import json
import math
import queue
import re
@@ -15,6 +16,7 @@ from fastapi.staticfiles import StaticFiles
from . import rebuild as rebuild_policy
from .config import Config
from .lyrics import normalize_lyrics
from .pipeline import (STAGE_ORDER, PipelineError, SongInput, run_pipeline,
song_slug)
@@ -303,6 +305,38 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
"video_filename": inp.video_path.name if inp.video_path else None,
"cover_filename": inp.cover_path.name if inp.cover_path else None}
@app.get("/api/jobs/{job_id}/review")
def get_review(job_id: str):
job = _finished_job(job_id)
if job.status != "done":
raise HTTPException(404)
work = cfg.work_dir / song_slug(job.inp)
notes_file = work / "notes_sec.json"
if not notes_file.exists():
raise HTTPException(404)
notes_data = json.loads(notes_file.read_text())
meta_file = work / "align_meta.json"
meta = json.loads(meta_file.read_text()) if meta_file.exists() else {}
dropped = set(meta.get("dropped_lines", []))
missing_spans = [{"start": s, "end": e}
for s, e in meta.get("missing_spans", [])]
align_file = work / "alignment.json"
align_rows = json.loads(align_file.read_text()) if align_file.exists() else []
words_by_line: dict[int, list] = {}
for r in align_rows:
words_by_line.setdefault(r["line_index"], []).append(
{"text": r["word"], "start": r["start"], "end": r["end"]})
ref = normalize_lyrics(job.inp.lyrics_text)
by_line: dict[int, list] = {}
for n in notes_data["notes"]:
by_line.setdefault(n["line_index"], []).append({
"start": n["start"], "end": n["start"] + n["duration"],
"pitch": n["pitch"], "low_confidence": n["low_confidence"]})
lines = [{"index": i, "text": " ".join(words), "dropped": i in dropped,
"notes": by_line.get(i, []), "words": words_by_line.get(i, [])}
for i, words in enumerate(ref.lines)]
return {"lines": lines, "missing_spans": missing_spans}
return app
+23
View File
@@ -137,3 +137,26 @@ def test_words_split_across_sections(tmp_path, monkeypatch):
check_invariants(rows, LYRICS)
assert rows[3]["end"] <= 5.0
assert rows[4]["start"] >= 29.0
def test_align_drops_lines_never_sung(tmp_path, monkeypatch):
lyrics = "she's got a smile\nthat it seems to me\nbut then it fades away"
transcript = [{"start": 1.0, "end": 4.0,
"text": "she's got a smile that it seems to me"}]
rows = run_align(tmp_path, monkeypatch, lyrics, transcript)
assert {r["line_index"] for r in rows} == {0, 1}
meta = json.loads((tmp_path / "w" / "align_meta.json").read_text())
assert meta["dropped_lines"] == [2]
assert meta["missing_spans"] == []
def test_align_flags_missing_span_for_unmatched_singing(tmp_path, monkeypatch):
lyrics = "she's got a smile\nthat it seems to me"
transcript = [
{"start": 1.0, "end": 4.0, "text": "she's got a smile that it seems to me"},
{"start": 20.0, "end": 23.0, "text": "completely unrelated ad lib bridge"},
]
run_align(tmp_path, monkeypatch, lyrics, transcript)
meta = json.loads((tmp_path / "w" / "align_meta.json").read_text())
assert meta["dropped_lines"] == []
assert meta["missing_spans"] == [[19.0, 24.0]]
+52
View File
@@ -295,3 +295,55 @@ def test_notes_stage_writes_json(tmp_path):
def test_notes_stage_registered():
assert "notes" in [s.name for s in all_stages()]
def test_build_notes_skips_notes_within_missing_span():
times, f0, conf = flat_pitch()
rows = [
{"word": "keep", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0},
{"word": "skip", "start": 5.0, "end": 5.4, "score": 0.9, "line_index": 1},
]
notes, _, _ = build_notes(rows, times, f0, conf, "English",
missing_spans=[(4.5, 6.0)])
assert [n.text for n in notes] == ["keep"]
def test_warnings_report_dropped_lines():
times, f0, conf = flat_pitch()
rows = [{"word": "keep", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0}]
ref_lines = [["keep"], ["the", "outro"]]
_, _, warnings = build_notes(rows, times, f0, conf, "English",
dropped_lines=[1], ref_lines=ref_lines)
assert warnings == ['1 line dropped (not sung) near "the outro"']
def test_warnings_report_missing_spans():
times, f0, conf = flat_pitch()
rows = [{"word": "keep", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0}]
_, _, warnings = build_notes(rows, times, f0, conf, "English",
missing_spans=[(81.0, 89.5)])
assert warnings == ["missing lyrics near 1:21–1:29"]
def test_build_notes_skipped_row_mid_line_does_not_corrupt_spacing_or_breaks():
# "gone" falls inside a missing span and is skipped; "away" is the same
# line's next word and must still get its leading space, and the break
# before the next line must use the last *emitted* word's end (3.4, from
# "away"), not the skipped row's end (2.4).
times, f0, conf = flat_pitch()
rows = [
{"word": "keep", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0},
{"word": "gone", "start": 2.0, "end": 2.4, "score": 0.9, "line_index": 0},
{"word": "away", "start": 3.0, "end": 3.4, "score": 0.9, "line_index": 0},
{"word": "next", "start": 5.0, "end": 5.4, "score": 0.9, "line_index": 1},
]
notes, breaks, _ = build_notes(rows, times, f0, conf, "English",
missing_spans=[(1.9, 2.5)])
assert [n.text for n in notes] == ["keep", " away", "next"]
assert breaks == [(3.4 + 5.0) / 2]
def test_notes_carry_line_index():
times, f0, conf = flat_pitch()
rows = [{"word": "hey", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 3}]
notes, _, _ = build_notes(rows, times, f0, conf, "English")
assert notes[0].line_index == 3
+39 -1
View File
@@ -1,6 +1,6 @@
from songtrace.lyrics import normalize_lyrics
from songtrace.whisper_client import TranscriptSegment
from songtrace.stages.transcribe import lyrics_similarity, vocal_sections
from songtrace.stages.transcribe import classify_lyrics_match, lyrics_similarity, vocal_sections
def seg(s, e, t=""):
return TranscriptSegment(start=s, end=e, text=t)
@@ -26,3 +26,41 @@ def test_similarity_low_for_wrong_lyrics():
ref = normalize_lyrics("completely different words here entirely")
segs = [seg(0, 2, "she's got a smile that it seems to me")]
assert lyrics_similarity(segs, ref) < 0.4
def test_classify_lyrics_match_flags_extra_reference_line():
ref = normalize_lyrics("she's got a smile\nthat it seems to me\nbut then it fades")
segs = [seg(1, 4, "she's got a smile that it seems to me")]
dropped, missing = classify_lyrics_match(segs, ref)
assert dropped == [2]
assert missing == []
def test_classify_lyrics_match_flags_unmatched_singing():
ref = normalize_lyrics("she's got a smile\nthat it seems to me")
segs = [seg(1, 4, "she's got a smile that it seems to me"),
seg(20, 23, "completely unrelated ad lib bridge")]
dropped, missing = classify_lyrics_match(segs, ref)
assert dropped == []
assert missing == [(19.0, 24.0)]
def test_classify_lyrics_match_keeps_partially_matched_lines():
ref = normalize_lyrics("she's got a smile\nthat it seems to me")
# line 1 heard with one word garbled ("it" -> "bit"): still a replace/equal
# match, so the line stays matched rather than dropped
segs = [seg(1, 4, "she's got a smile that bit seems to me")]
dropped, _ = classify_lyrics_match(segs, ref)
assert dropped == []
def test_classify_lyrics_match_scopes_missing_span_to_adlib_not_whole_section():
# matched line and an unmatched ad-lib close enough together that
# vocal_sections would merge them into one section; missing_spans must
# stay tight around the ad-lib so the matched line keeps its notes.
ref = normalize_lyrics("she's got a smile\nthat it seems to me")
segs = [seg(1, 4, "she's got a smile that it seems to me"),
seg(6, 8, "completely unrelated ad lib")]
sections = vocal_sections(segs)
assert sections == [(0.0, 9.0)] # confirms the two segments do merge
dropped, missing = classify_lyrics_match(segs, ref)
assert dropped == []
assert missing == [(5.0, 9.0)]
for a, b in missing:
assert not (a <= 1.0 < b) # matched line's start (1.0) untouched
+185 -3
View File
@@ -1,4 +1,5 @@
import io
import json
import re
import threading
import time
@@ -359,6 +360,32 @@ def test_index_has_required_hooks():
assert f'name="{name}"' in html, name
def test_video_source_fields_are_outside_advanced_details():
html = _index_html()
form_start = html.index('id="song-form"')
details_start = html.index("<details>", form_start)
before = html[form_start:details_start]
assert 'name="youtube_url"' in before
assert 'name="video"' in before
after = html[details_start:]
for name in ("genre", "year", "language", "bpm", "cover"):
assert f'name="{name}"' in after
def test_song_form_lyrics_uses_a_two_column_grid():
css = _page_css(_index_html())
assert re.search(r"\.song-form-grid\s*\{[^}]*grid-template-columns:\s*1fr 1fr",
css, re.S)
def test_song_form_grid_stacks_on_narrow_screens():
css = _page_css(_index_html())
assert re.search(
r"@media \(max-width: 720px\)\s*\{[^}]*\.song-form-grid\s*\{[^}]*"
r"grid-template-columns:\s*1fr\s*;",
css, re.S)
def test_index_makes_no_external_requests():
import re as _re
html = _index_html()
@@ -383,10 +410,53 @@ def test_index_respects_reduced_motion():
assert "prefers-reduced-motion" in _index_html()
def test_index_parses_warnings_with_endash():
def test_index_has_lyrics_review_hook():
html = _index_html()
assert "–" in html, "warning parser must handle the en-dash separator"
assert "data-start" in html and "data-end" in html
assert 'id="lyrics-review"' in html
assert 'id="warnings"' not in html
def test_index_review_uses_svg_not_canvas():
html = _index_html()
assert "createElementNS" in html
assert "<canvas" not in html
def test_index_review_fetches_review_endpoint():
html = _index_html()
assert "/review" in html
def test_index_lyrics_review_uses_endash_for_time_ranges():
html = _index_html()
body = _fn_body(html, "missingSpanNode")
assert "–" in body
def test_index_review_lays_out_words_in_time():
"""Words must be positioned by their sung timing (percentage-based), not
packed as plain flow text, so notes can line up under their own word."""
html = _index_html()
body = _fn_body(html, "lineTimeline")
assert "review-word" in body
assert "style.left" in body
def test_lyrics_review_cache_clears_on_job_status_change():
"""Regression: switching between done/failed/running jobs must clear the
cache so a previously-viewed done job re-fetches when re-selected."""
html = _index_html()
body = _fn_body(html, "renderLyricsReview")
# The "failed" branch must clear the cache before rendering the placeholder
assert re.search(
r'if\s*\(\s*job\.status\s*===\s*"failed"\s*\)\s*\{[^}]*'
r'delete\s+box\.dataset\.jobId',
body, re.S), "failed branch must delete cache key"
# The "not done" (queued/running) branch must clear the cache before rendering
assert re.search(
r'if\s*\(\s*job\.status\s*!==\s*"done"\s*\)\s*\{[^}]*'
r'delete\s+box\.dataset\.jobId',
body, re.S), "not done branch must delete cache key"
def test_index_has_copy_path_control():
@@ -958,3 +1028,115 @@ def test_a_plain_job_never_removes_anything(tmp_path):
stray.mkdir(parents=True)
make_done_job(client, title="T2")
assert stray.exists()
def review_runner_factory(notes, dropped_lines=None, missing_spans=None, warnings=None,
words=None):
def runner(inp, cfg, log=print, force_from=None, stages=None):
work = cfg.work_dir / song_slug(inp)
work.mkdir(parents=True, exist_ok=True)
(work / "notes_sec.json").write_text(json.dumps({
"notes": notes, "breaks": [], "warnings": warnings or []}))
if dropped_lines is not None or missing_spans is not None:
(work / "align_meta.json").write_text(json.dumps({
"dropped_lines": dropped_lines or [],
"missing_spans": missing_spans or []}))
if words is not None:
(work / "alignment.json").write_text(json.dumps(words))
out = cfg.output_dir / song_slug(inp)
out.mkdir(parents=True, exist_ok=True)
return PipelineResult(output_dir=out, warnings=warnings or [])
return runner
def test_review_endpoint_groups_notes_by_line(tmp_path):
notes = [
{"start": 1.0, "duration": 0.4, "pitch": 0, "text": "hi",
"low_confidence": False, "line_index": 0},
{"start": 5.0, "duration": 0.3, "pitch": 2, "text": "there",
"low_confidence": True, "line_index": 1},
]
client, _ = make_client(tmp_path, review_runner_factory(
notes, dropped_lines=[2], missing_spans=[[8.0, 9.5]]))
resp = client.post("/api/jobs", data={
"title": "T", "artist": "A", "lyrics": "hi\nthere\ngone",
"youtube_url": "http://y"})
job_id = resp.json()["id"]
wait_status(client, job_id, "done")
body = client.get(f"/api/jobs/{job_id}/review").json()
assert body["lines"] == [
{"index": 0, "text": "hi", "dropped": False,
"notes": [{"start": 1.0, "end": 1.4, "pitch": 0, "low_confidence": False}],
"words": []},
{"index": 1, "text": "there", "dropped": False,
"notes": [{"start": 5.0, "end": 5.3, "pitch": 2, "low_confidence": True}],
"words": []},
{"index": 2, "text": "gone", "dropped": True, "notes": [], "words": []},
]
assert body["missing_spans"] == [{"start": 8.0, "end": 9.5}]
def test_review_endpoint_includes_per_word_timing(tmp_path):
notes = [
{"start": 1.0, "duration": 0.4, "pitch": 0, "text": "hi",
"low_confidence": False, "line_index": 0},
{"start": 5.0, "duration": 0.3, "pitch": 2, "text": "there",
"low_confidence": True, "line_index": 1},
]
words = [
{"word": "hi", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0},
{"word": "there", "start": 5.0, "end": 5.3, "score": 0.9, "line_index": 1},
]
client, _ = make_client(tmp_path, review_runner_factory(
notes, dropped_lines=[2], missing_spans=[[8.0, 9.5]], words=words))
resp = client.post("/api/jobs", data={
"title": "T", "artist": "A", "lyrics": "hi\nthere\ngone",
"youtube_url": "http://y"})
job_id = resp.json()["id"]
wait_status(client, job_id, "done")
body = client.get(f"/api/jobs/{job_id}/review").json()
assert body["lines"][0]["words"] == [{"text": "hi", "start": 1.0, "end": 1.4}]
assert body["lines"][1]["words"] == [{"text": "there", "start": 5.0, "end": 5.3}]
assert body["lines"][2]["dropped"] is True
assert body["lines"][2]["words"] == []
def test_review_of_a_failed_job_is_404(tmp_path):
def bad_runner(*a, **kw):
raise PipelineError("boom")
client, _ = make_client(tmp_path, bad_runner)
resp = client.post("/api/jobs", data={
"title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"})
job_id = resp.json()["id"]
wait_status(client, job_id, "failed")
assert client.get(f"/api/jobs/{job_id}/review").status_code == 404
def test_review_of_a_running_job_is_409(tmp_path):
release = threading.Event()
def blocking(inp, cfg, log=print, force_from=None, stages=None):
release.wait(5)
return PipelineResult(output_dir=cfg.output_dir / "X")
client, _ = make_client(tmp_path, blocking)
resp = client.post("/api/jobs", data={
"title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"})
job_id = resp.json()["id"]
try:
wait_status(client, job_id, "running")
assert client.get(f"/api/jobs/{job_id}/review").status_code == 409
finally:
release.set()
wait_status(client, job_id, "done")
def test_review_of_an_unknown_job_is_404(tmp_path):
client, _ = make_client(tmp_path, ok_runner)
assert client.get("/api/jobs/nope/review").status_code == 404
def test_review_missing_when_pipeline_never_wrote_notes(tmp_path):
client, _ = make_client(tmp_path, ok_runner)
job_id = make_done_job(client)
assert client.get(f"/api/jobs/{job_id}/review").status_code == 404