branderson/songtrace
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases Wiki Activity

review: lay out lyrics words along a shared time axis with notes aligned beneath

Browse Source 
The review panel used to show a full line of packed text with one
undifferentiated strip of notes below it, so it was impossible to tell
which notes belonged to which word. The forced-alignment stage already
produces per-word timestamps in alignment.json, but they were never
surfaced past the pipeline.

/api/jobs/{id}/review now includes each line's per-word timing, and the
frontend spaces words out by their real sung timing (karaoke-style)
instead of normal text flow, with the note strip sharing that same time
axis so each note lands directly under its word. Alternating lane
backgrounds make the word/note pairing unambiguous. Dropped lines and
jobs without alignment data keep the old plain-text rendering.
This commit is contained in:
branderson
2026-07-28 01:22:48 -07:00
parent 0599f6b665
commit 5023ea69d3
5 changed files with 123 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/images/webui.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 77 KiB

After

Width:  |  Height:  |  Size: 77 KiB

scripts/screenshot_webui.py
+20 -3
View File
@@ -70,12 +70,24 @@ SONGS = [
{"index": 0, "text": "Alas, my love, you do me wrong",
"dropped": False,
"notes": [{"start": 12.0, "end": 12.4, "pitch": 0, "low_confidence": False},
{"start": 12.5, "end": 12.9, "pitch": 2, "low_confidence": False}]},
{"start": 12.5, "end": 12.9, "pitch": 2, "low_confidence": False}],
"words": [{"text": "Alas,", "start": 10.0, "end": 10.4},
{"text": "my", "start": 10.4, "end": 10.6},
{"text": "love,", "start": 10.6, "end": 11.0},
{"text": "you", "start": 11.0, "end": 11.2},
{"text": "do", "start": 11.2, "end": 11.4},
{"text": "me", "start": 11.4, "end": 11.6},
{"text": "wrong", "start": 11.8, "end": 12.9}]},
{"index": 1, "text": "To cast me off discourteously",
"dropped": False,
"notes": [{"start": 45.0, "end": 45.4, "pitch": 4, "low_confidence": True}]},
"notes": [{"start": 45.0, "end": 45.4, "pitch": 4, "low_confidence": True}],
"words": [{"text": "To", "start": 44.0, "end": 44.3},
{"text": "cast", "start": 44.3, "end": 44.6},
{"text": "me", "start": 44.6, "end": 44.8},
{"text": "off", "start": 44.8, "end": 45.0},
{"text": "discourteously", "start": 45.0, "end": 45.4}]},
{"index": 2, "text": "For I have loved you well and long",
"dropped": True, "notes": []},
"dropped": True, "notes": [], "words": []},
],
"missing_spans": [{"start": 130.0, "end": 138.0}],
}}),
@@ -113,6 +125,11 @@ def make_runner():
"missing_spans": [[s["start"], s["end"]]
for s in extra["review"]["missing_spans"]],
}))
(work / "alignment.json").write_text(json.dumps([
{"word": w["text"], "start": w["start"], "end": w["end"],
"score": 0.9, "line_index": line["index"]}
for line in extra["review"]["lines"] for w in line.get("words", [])
]))
out = cfg.output_dir / f"{inp.artist} - {inp.title}"
out.mkdir(parents=True, exist_ok=True)
return PipelineResult(output_dir=out, warnings=extra["warnings"])
src/songtrace/static/index.html
+53 -9
View File
@@ -325,7 +325,12 @@ button { cursor: pointer; }
.review-line-dropped { opacity: 0.55; }
.review-line-dropped .review-line-text { text-decoration: line-through; }
.review-line-note { display: block; margin-top: 4px; font-size: 0.72rem; color: var(--text-muted); }
.review-roll { display: block; width: 100%; height: 20px; margin-top: 6px; }
.review-line-timeline { position: relative; margin-top: 4px; }
.review-lane { position: absolute; top: 0; bottom: 0; }
.review-lane-alt { background: var(--panel-raised); }
.review-word-row { position: relative; height: 18px; }
.review-word { position: absolute; top: 0; white-space: nowrap; font-size: 0.9rem; }
.review-roll { display: block; width: 100%; height: 20px; margin-top: 4px; }
.review-note { fill: var(--aqua); }
.review-note-low { fill: var(--warn); }
.review-missing {
@@ -804,13 +809,11 @@ function missingSpanNode(span) {
return p;
}
function pianoRoll(notes, minPitch, span) {
function pianoRoll(notes, minPitch, span, t0, t1) {
const svg = document.createElementNS(SVG_NS, "svg");
svg.setAttribute("class", "review-roll");
svg.setAttribute("viewBox", "0 0 100 20");
svg.setAttribute("preserveAspectRatio", "none");
const t0 = notes[0].start;
const t1 = Math.max(...notes.map(n => n.end));
const dur = Math.max(t1 - t0, 0.001);
for (const n of notes) {
const rect = document.createElementNS(SVG_NS, "rect");
@@ -827,21 +830,62 @@ function pianoRoll(notes, minPitch, span) {
return svg;
}
// Lays words out along a shared time axis (like a karaoke lyric line) so
// each word's notes render directly beneath it instead of as one flat strip.
function lineTimeline(words, notes, minPitch, span) {
const t0 = words[0].start;
const t1 = words[words.length - 1].end;
const dur = Math.max(t1 - t0, 0.001);
const box = document.createElement("div");
box.className = "review-line-timeline";
words.forEach((w, i) => {
const lane = document.createElement("div");
lane.className = "review-lane" + (i % 2 ? " review-lane-alt" : "");
lane.style.left = `${((w.start - t0) / dur) * 100}%`;
lane.style.width = `${Math.max(((w.end - w.start) / dur) * 100, 0.5)}%`;
box.append(lane);
});
const wordsRow = document.createElement("div");
wordsRow.className = "review-word-row";
for (const w of words) {
const label = document.createElement("span");
label.className = "review-word";
label.style.left = `${((w.start - t0) / dur) * 100}%`;
label.textContent = w.text;
wordsRow.append(label);
}
box.append(wordsRow);
if (notes.length) box.append(pianoRoll(notes, minPitch, span, t0, t1));
return box;
}
function lineNode(line, minPitch, span) {
const row = document.createElement("div");
row.className = "review-line" + (line.dropped ? " review-line-dropped" : "");
const text = document.createElement("span");
text.className = "review-line-text";
text.textContent = line.text;
row.append(text);
if (line.dropped) {
const text = document.createElement("span");
text.className = "review-line-text";
text.textContent = line.text;
row.append(text);
const note = document.createElement("span");
note.className = "review-line-note";
note.textContent = "not sung — dropped";
row.append(note);
return row;
}
if (line.notes.length) row.append(pianoRoll(line.notes, minPitch, span));
if (line.words.length) {
row.append(lineTimeline(line.words, line.notes, minPitch, span));
return row;
}
const text = document.createElement("span");
text.className = "review-line-text";
text.textContent = line.text;
row.append(text);
if (line.notes.length) {
const t0 = line.notes[0].start;
const t1 = Math.max(...line.notes.map(n => n.end));
row.append(pianoRoll(line.notes, minPitch, span, t0, t1));
}
return row;
}
src/songtrace/web.py
+7 -1
View File
@@ -320,6 +320,12 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
dropped = set(meta.get("dropped_lines", []))
missing_spans = [{"start": s, "end": e}
for s, e in meta.get("missing_spans", [])]
align_file = work / "alignment.json"
align_rows = json.loads(align_file.read_text()) if align_file.exists() else []
words_by_line: dict[int, list] = {}
for r in align_rows:
words_by_line.setdefault(r["line_index"], []).append(
{"text": r["word"], "start": r["start"], "end": r["end"]})
ref = normalize_lyrics(job.inp.lyrics_text)
by_line: dict[int, list] = {}
for n in notes_data["notes"]:
@@ -327,7 +333,7 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
"start": n["start"], "end": n["start"] + n["duration"],
"pitch": n["pitch"], "low_confidence": n["low_confidence"]})
lines = [{"index": i, "text": " ".join(words), "dropped": i in dropped,
"notes": by_line.get(i, [])}
"notes": by_line.get(i, []), "words": words_by_line.get(i, [])}
for i, words in enumerate(ref.lines)]
return {"lines": lines, "missing_spans": missing_spans}
tests/test_web.py
+43 -4
View File
@@ -433,6 +433,15 @@ def test_index_lyrics_review_uses_endash_for_time_ranges():
assert "–" in body
def test_index_review_lays_out_words_in_time():
"""Words must be positioned by their sung timing (percentage-based), not
packed as plain flow text, so notes can line up under their own word."""
html = _index_html()
body = _fn_body(html, "lineTimeline")
assert "review-word" in body
assert "style.left" in body
def test_lyrics_review_cache_clears_on_job_status_change():
"""Regression: switching between done/failed/running jobs must clear the
cache so a previously-viewed done job re-fetches when re-selected."""
@@ -1021,7 +1030,8 @@ def test_a_plain_job_never_removes_anything(tmp_path):
assert stray.exists()
def review_runner_factory(notes, dropped_lines=None, missing_spans=None, warnings=None):
def review_runner_factory(notes, dropped_lines=None, missing_spans=None, warnings=None,
words=None):
def runner(inp, cfg, log=print, force_from=None, stages=None):
work = cfg.work_dir / song_slug(inp)
work.mkdir(parents=True, exist_ok=True)
@@ -1031,6 +1041,8 @@ def review_runner_factory(notes, dropped_lines=None, missing_spans=None, warning
(work / "align_meta.json").write_text(json.dumps({
"dropped_lines": dropped_lines or [],
"missing_spans": missing_spans or []}))
if words is not None:
(work / "alignment.json").write_text(json.dumps(words))
out = cfg.output_dir / song_slug(inp)
out.mkdir(parents=True, exist_ok=True)
return PipelineResult(output_dir=out, warnings=warnings or [])
@@ -1054,14 +1066,41 @@ def test_review_endpoint_groups_notes_by_line(tmp_path):
body = client.get(f"/api/jobs/{job_id}/review").json()
assert body["lines"] == [
{"index": 0, "text": "hi", "dropped": False,
"notes": [{"start": 1.0, "end": 1.4, "pitch": 0, "low_confidence": False}]},
"notes": [{"start": 1.0, "end": 1.4, "pitch": 0, "low_confidence": False}],
"words": []},
{"index": 1, "text": "there", "dropped": False,
"notes": [{"start": 5.0, "end": 5.3, "pitch": 2, "low_confidence": True}]},
{"index": 2, "text": "gone", "dropped": True, "notes": []},
"notes": [{"start": 5.0, "end": 5.3, "pitch": 2, "low_confidence": True}],
"words": []},
{"index": 2, "text": "gone", "dropped": True, "notes": [], "words": []},
]
assert body["missing_spans"] == [{"start": 8.0, "end": 9.5}]
def test_review_endpoint_includes_per_word_timing(tmp_path):
notes = [
{"start": 1.0, "duration": 0.4, "pitch": 0, "text": "hi",
"low_confidence": False, "line_index": 0},
{"start": 5.0, "duration": 0.3, "pitch": 2, "text": "there",
"low_confidence": True, "line_index": 1},
]
words = [
{"word": "hi", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0},
{"word": "there", "start": 5.0, "end": 5.3, "score": 0.9, "line_index": 1},
]
client, _ = make_client(tmp_path, review_runner_factory(
notes, dropped_lines=[2], missing_spans=[[8.0, 9.5]], words=words))
resp = client.post("/api/jobs", data={
"title": "T", "artist": "A", "lyrics": "hi\nthere\ngone",
"youtube_url": "http://y"})
job_id = resp.json()["id"]
wait_status(client, job_id, "done")
body = client.get(f"/api/jobs/{job_id}/review").json()
assert body["lines"][0]["words"] == [{"text": "hi", "start": 1.0, "end": 1.4}]
assert body["lines"][1]["words"] == [{"text": "there", "start": 5.0, "end": 5.3}]
assert body["lines"][2]["dropped"] is True
assert body["lines"][2]["words"] == []
def test_review_of_a_failed_job_is_404(tmp_path):
def bad_runner(*a, **kw):
raise PipelineError("boom")