webui: a note sheet with a real pitch axis and beat grid
tests / test (pull_request_target) Successful in 4m0s

The review drew each line's notes as thin bars in a 20px strip, with no
pitch axis or beat grid, so it was hard to tell where a note sat. Add a
sheet that shows one line at a time the way an UltraStar editor does: a
row per semitone, shaded like piano keys and named in a gutter, the
chart's own beat grid with bars every 4 and 16 beats, notes as blocks
styled by kind (golden, freestyle, rap) with low-confidence ones outlined,
neighbouring lines faded in for context, syllables and aligned words
beneath, and a playhead that follows the transport, paging with it.

The detected pitch curve is drawn over the notes, served by a new GET
/api/jobs/{id}/pitch in note semitones, so a note off the sung pitch
stands out. The sheet is its own ES module; its geometry is unit-tested
with node --test, run from pytest and skipped where Node is absent.
This commit is contained in:
2026-10-11 01:19:12 -07:00
parent cde56826a1
commit e30babe287
6 changed files with 455 additions and 0 deletions
+80
View File
@@ -392,6 +392,67 @@ button { cursor: pointer; }
/* the native transport renders light otherwise, against everything around it */
#player-media { display: block; width: 100%; margin-top: 8px; color-scheme: dark; }
#sheet { margin: 14px 0; padding: 12px 14px 10px; background: var(--panel); border: 1px solid var(--border); }
#sheet[hidden] { display: none; }
.sheet-nav { display: flex; align-items: center; gap: 12px; margin-bottom: 12px; }
.sheet-title {
min-width: 9em;
text-align: center;
font-family: var(--font-display);
font-weight: 700;
font-size: 0.72rem;
letter-spacing: 0.18em;
color: var(--text-muted);
}
.sheet-nav .action:disabled { opacity: 0.35; cursor: default; }
.sheet-badges { display: flex; gap: 8px; margin-left: auto; }
.sheet-badge {
padding: 2px 8px;
font-family: var(--font-display);
font-weight: 600;
font-size: 0.66rem;
letter-spacing: 0.1em;
text-transform: uppercase;
border: 1px solid currentColor;
border-radius: 2px;
}
.sheet-badge-edited { color: var(--aqua); }
.sheet-badge-low { color: var(--fail); }
.sheet-plot { position: relative; cursor: crosshair; }
.sheet-svg { display: block; }
.sheet-row { fill: var(--panel-raised); }
.sheet-row-black { fill: var(--bg); }
.sheet-octave { stroke: var(--control); stroke-width: 1; }
.sheet-key { fill: var(--text-muted); font: 10px var(--font-mono); text-anchor: end; dominant-baseline: middle; }
.sheet-key-c { fill: var(--text); font-weight: 600; }
.sheet-beat { stroke: var(--border); stroke-width: 1; opacity: 0.35; }
.sheet-beat-1 { stroke: var(--control); opacity: 0.55; }
.sheet-beat-2 { stroke: var(--text-muted); opacity: 0.8; }
.sheet-curve { pointer-events: none; }
.sheet-curve polyline { fill: none; stroke: var(--aqua); stroke-width: 1.5; opacity: 0.9; }
.sheet-note { fill: var(--purple); }
.sheet-note-golden { fill: #E8B931; }
.sheet-note-free { fill: none; stroke: var(--purple); stroke-width: 1.5; stroke-dasharray: 4 3; }
.sheet-note-rap { fill: var(--aqua); }
.sheet-note-golden-rap { fill: #E8B931; stroke: var(--aqua); stroke-width: 2; }
.sheet-note-low { stroke: var(--fail); stroke-width: 2; stroke-dasharray: 3 2; }
.sheet-note-context { opacity: 0.28; }
.sheet-playhead { position: absolute; top: 0; height: 300px; width: 2px; margin-left: -1px; background: var(--aqua); pointer-events: none; }
.sheet-playhead[hidden] { display: none; }
.sheet-syllables, .sheet-words { position: relative; overflow: hidden; }
.sheet-syllables { height: 20px; margin-top: 6px; }
.sheet-syllable { position: absolute; top: 0; font-size: 0.8rem; white-space: pre; color: var(--text); }
.sheet-words { height: 24px; margin-top: 4px; }
.sheet-word {
position: absolute;
top: 0;
padding-top: 4px;
font-size: 0.78rem;
white-space: nowrap;
color: var(--text-muted);
border-top: 2px solid var(--border);
}
#lyrics-review { display: flex; flex-direction: column; gap: 10px; }
.review-line { padding: 8px 12px; background: var(--panel); border: 1px solid var(--border); }
.review-line-active { background: var(--panel-raised); border-color: var(--aqua); }
@@ -658,6 +719,7 @@ button { cursor: pointer; }
<video id="player-video" muted playsinline preload="metadata"></video>
<audio id="player-media" controls preload="metadata"></audio>
</div>
<div id="sheet" hidden></div>
<div id="lyrics-review"><p class="empty-note">no songs queued yet — nothing to review</p></div>
<div id="output-path"></div>
<div id="review-actions"></div>
@@ -1151,6 +1213,7 @@ function lineEntry(row, line) {
const labels = [...row.querySelectorAll(".review-word")];
return {
row,
index: line.index,
t0: laid[0].start,
t1: Math.max(...laid.map(x => x.end)),
words: line.words.map((w, i) => ({ el: labels[i], start: w.start, end: w.end })),
@@ -1197,6 +1260,17 @@ const media = document.getElementById("player-media");
const video = document.getElementById("player-video");
const DRIFT = 0.25; // seconds before the picture is worth re-seeking
// the note sheet lives in its own module, loaded before any review renders
let sheet = null;
const sheetReady = import("/static/sheet.js").then((m) => {
sheet = m.createSheet(document.getElementById("sheet"), {
seek: (t) => {
media.currentTime = t;
if (media.paused) syncPlayback();
},
});
});
function slaveVideo(force) {
if (video.hidden || !video.getAttribute("src")) return;
if (force || Math.abs(video.currentTime - media.currentTime) > DRIFT) {
@@ -1251,10 +1325,12 @@ function syncPlayback() {
activeEntry = entry;
if (entry) {
entry.row.classList.add("review-line-active");
sheet?.showLine(entry.index);
focusActive();
}
}
if (entry) paintEntry(entry, t);
sheet?.setTime(t);
}
// centred so there is context either side, like a scrolling lyric sheet; only
@@ -1314,6 +1390,7 @@ function setPlayerSource(job, available) {
}
function resetPlayer() {
sheet?.clear();
// the 3s poll reaches here for every running job; reloading a cleared
// element each time would be pure churn
if (player.hidden && !timeline.length) return;
@@ -1361,6 +1438,9 @@ async function renderLyricsReview(job) {
}
const { nodes, entries } = buildLyricsReview(data);
box.replaceChildren(...nodes);
await sheetReady;
if (job.id !== selectedId) return;
sheet.load(job.id, data);
// with nothing to play, a highlighted line and a click-to-seek cursor would
// both be promising something the song folder cannot deliver
const playable = setPlayerSource(job, data.media);
+265
View File
@@ -0,0 +1,265 @@
// The note sheet: one lyric line at a time on a pitch-by-beat grid, the way
// an UltraStar editor lays a song out, with the detected pitch drawn behind
// the notes so a wrong one stands out.
const SVG = "http://www.w3.org/2000/svg";
const NAMES = ["C", "C♯", "D", "D♯", "E", "F", "F♯", "G", "G♯", "A", "A♯", "B"];
const BLACK = new Set([1, 3, 6, 8, 10]);
const GUTTER = 46; // px of note names left of the grid
const PLOT_HEIGHT = 300; // px; rows divide this between the pitches shown
const CONTEXT = 0.75; // seconds shown either side of the line
const wrap = (n, d) => ((n % d) + d) % d;
const mod12 = (p) => wrap(p, 12);
// UltraStar pitch 0 is C4 (MIDI 60)
export function noteName(pitch) {
return NAMES[mod12(pitch)] + (Math.floor((pitch + 60) / 12) - 1);
}
export function isBlackKey(pitch) {
return BLACK.has(mod12(pitch));
}
// The pitches to show: the line's notes with room around them, and never
// less than an octave and a bit, so rows stay a consistent size to aim at.
export function pitchRange(pitches, { pad = 3, min = 15 } = {}) {
if (!pitches.length) return { lo: -7, hi: 7 };
let lo = Math.min(...pitches) - pad;
let hi = Math.max(...pitches) + pad;
const short = min - (hi - lo + 1);
if (short > 0) {
lo -= Math.floor(short / 2);
hi += Math.ceil(short / 2);
}
return { lo, hi };
}
// UltraStar beats between t0 and t1 on the chart's grid. A beat is a quarter
// of the #BPM beat; weight 2 marks every 16th, 1 every 4th, as editors do.
export function beatLines(t0, t1, grid) {
const spb = 60 / (4 * grid.bpm);
const gap = grid.gap_ms / 1000;
const out = [];
for (let k = Math.ceil((t0 - gap) / spb); gap + k * spb <= t1; k++) {
const weight = wrap(k, 16) === 0 ? 2 : wrap(k, 4) === 0 ? 1 : 0;
out.push({ t: gap + k * spb, beat: k, weight });
}
return out;
}
// The seconds a line occupies, from its notes and the words it was aligned to.
export function lineSpan(line) {
const ts = [...line.notes.flatMap(n => [n.start, n.end]),
...line.words.flatMap(w => [w.start, w.end])];
return ts.length ? { start: Math.min(...ts), end: Math.max(...ts) } : null;
}
function el(name, attrs = {}, parent = null) {
const node = document.createElementNS(SVG, name);
for (const [k, v] of Object.entries(attrs)) node.setAttribute(k, v);
if (parent) parent.append(node);
return node;
}
function div(cls, text = null) {
const node = document.createElement("div");
node.className = cls;
if (text !== null) node.textContent = text;
return node;
}
export function createSheet(root, { seek }) {
const title = div("sheet-title");
const badges = div("sheet-badges");
const prev = document.createElement("button");
const next = document.createElement("button");
for (const [b, label, step] of [[prev, "◀ PREV", -1], [next, "NEXT ▶", 1]]) {
b.type = "button";
b.className = "action";
b.textContent = label;
b.addEventListener("click", () => page(step));
}
const nav = div("sheet-nav");
nav.append(prev, title, next, badges);
const plot = div("sheet-plot");
const svg = el("svg", { class: "sheet-svg" });
const playhead = div("sheet-playhead");
playhead.hidden = true;
plot.append(svg, playhead);
const syllables = div("sheet-syllables");
const words = div("sheet-words");
root.replaceChildren(nav, plot, syllables, words);
let review = null;
let curve = null;
let index = null;
let view = null; // what the last render laid out: window and scale
let jobId = null;
const visible = () => review.lines.filter(l => !l.dropped && lineSpan(l));
function page(step) {
const lines = visible();
const at = lines.findIndex(l => l.index === index);
const to = lines[Math.min(Math.max(at + step, 0), lines.length - 1)];
if (to) showLine(to.index);
}
function showLine(i) {
if (!review || i === index) return;
index = i;
render();
}
function render() {
if (!review || index === null) return;
const line = review.lines[index];
const span = lineSpan(line);
const t0 = span.start - CONTEXT;
const t1 = span.end + CONTEXT;
const width = Math.max(plot.clientWidth, 320);
const { lo, hi } = pitchRange(line.notes.map(n => n.pitch));
const rowH = PLOT_HEIGHT / (hi - lo + 1);
const x = (t) => GUTTER + ((t - t0) / (t1 - t0)) * (width - GUTTER);
const y = (p) => (hi - p) * rowH; // top of the row for pitch p
view = { t0, t1, x, width };
svg.setAttribute("viewBox", `0 0 ${width} ${PLOT_HEIGHT}`);
svg.setAttribute("width", width);
svg.setAttribute("height", PLOT_HEIGHT);
svg.replaceChildren();
const rows = el("g", { class: "sheet-rows" }, svg);
for (let p = lo; p <= hi; p++) {
el("rect", { x: GUTTER, y: y(p), width: width - GUTTER, height: rowH,
class: isBlackKey(p) ? "sheet-row sheet-row-black" : "sheet-row" }, rows);
if (mod12(p) === 0) {
el("line", { x1: GUTTER, x2: width, y1: y(p) + rowH, y2: y(p) + rowH,
class: "sheet-octave" }, rows);
}
// every C names its octave; the naturals between get a name when
// there is room for it
if (mod12(p) === 0 || (rowH >= 16 && !isBlackKey(p))) {
const label = el("text", { x: GUTTER - 6, y: y(p) + rowH / 2,
class: mod12(p) === 0 ? "sheet-key sheet-key-c" : "sheet-key" },
rows);
label.textContent = noteName(p);
}
}
if (review.grid) {
const beats = el("g", { class: "sheet-beats" }, svg);
const spbPx = x(60 / (4 * review.grid.bpm)) - x(0);
for (const b of beatLines(t0, t1, review.grid)) {
if (b.weight === 0 && spbPx < 10) continue;
el("line", { x1: x(b.t), x2: x(b.t), y1: 0, y2: PLOT_HEIGHT,
class: `sheet-beat sheet-beat-${b.weight}` }, beats);
}
}
const notes = el("g", { class: "sheet-notes" }, svg);
const syllableNodes = [];
for (const l of review.lines) {
if (l.dropped) continue;
const current = l.index === index;
for (const n of l.notes) {
if (n.end < t0 || n.start > t1) continue;
const kind = { "*": "golden", F: "free", R: "rap", G: "golden-rap" }[n.kind] ?? "normal";
const cls = ["sheet-note", `sheet-note-${kind}`];
if (!current) cls.push("sheet-note-context");
if (current && n.low_confidence) cls.push("sheet-note-low");
const pad = Math.min(2, rowH * 0.12);
el("rect", { x: x(n.start), y: y(n.pitch) + pad,
width: Math.max(x(n.end) - x(n.start), 2), height: rowH - 2 * pad,
rx: Math.min(4, rowH / 3), class: cls.join(" ") }, notes)
.append(Object.assign(el("title"), {
textContent: `${n.text.trim() || "~"} · ${noteName(n.pitch)}` }));
if (current) {
const s = div("sheet-syllable", n.text.trim() || "~");
s.style.left = `${x(n.start)}px`;
syllableNodes.push(s);
}
}
}
syllables.replaceChildren(...syllableNodes);
// over the notes: the curve matters most where it agrees with one
if (curve) {
const g = el("g", { class: "sheet-curve" }, svg);
const from = Math.max(0, Math.floor((t0 - curve.start) / curve.step));
const to = Math.min(curve.pitch.length, Math.ceil((t1 - curve.start) / curve.step));
let pts = [];
const flush = () => {
if (pts.length > 1) el("polyline", { points: pts.join(" ") }, g);
pts = [];
};
for (let k = from; k < to; k++) {
const p = curve.pitch[k];
if (p === null || p < lo - 0.5 || p > hi + 0.5) { flush(); continue; }
pts.push(`${x(curve.start + k * curve.step).toFixed(1)},${(y(p) + rowH / 2).toFixed(1)}`);
}
flush();
}
words.replaceChildren(...line.words.map(w => {
const node = div("sheet-word", w.text);
node.style.left = `${x(w.start)}px`;
node.style.width = `${Math.max(x(w.end) - x(w.start), 2)}px`;
return node;
}));
const lines = visible();
const at = lines.findIndex(l => l.index === index);
title.textContent = `LINE ${at + 1} / ${lines.length}`;
prev.disabled = at <= 0;
next.disabled = at >= lines.length - 1;
const tags = [];
if (line.edited) tags.push(["edited", "sheet-badge-edited"]);
if (line.notes.some(n => n.low_confidence)) tags.push(["low confidence", "sheet-badge-low"]);
badges.replaceChildren(...tags.map(([text, cls]) => div(`sheet-badge ${cls}`, text)));
}
plot.addEventListener("click", (e) => {
if (!view) return;
const px = e.clientX - plot.getBoundingClientRect().left;
if (px < GUTTER) return;
seek(view.t0 + ((px - GUTTER) / (view.width - GUTTER)) * (view.t1 - view.t0));
});
new ResizeObserver(() => render()).observe(plot);
return {
load(id, data) {
review = data;
root.hidden = false;
const first = visible()[0];
index = null;
if (id !== jobId) {
jobId = id;
curve = null;
fetch(`/api/jobs/${id}/pitch`)
.then(r => (r.ok ? r.json() : null))
.then(c => { if (jobId === id) { curve = c; render(); } })
.catch(() => {});
}
if (first) showLine(first.index);
else root.hidden = true;
},
clear() {
review = null;
jobId = null;
curve = null;
index = null;
view = null;
root.hidden = true;
},
showLine,
setTime(t) {
if (!view) return;
const inside = t >= view.t0 && t <= view.t1;
playhead.hidden = !inside;
if (inside) playhead.style.left = `${view.x(t)}px`;
},
};
}
+22
View File
@@ -9,6 +9,7 @@ import uuid
from dataclasses import asdict, dataclass, field
from pathlib import Path
import numpy as np
from fastapi import Body, FastAPI, File, Form, HTTPException, UploadFile
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import FileResponse
@@ -18,6 +19,7 @@ from . import edits, grid
from . import rebuild as rebuild_policy
from .config import Config
from .lyrics import normalize_lyrics
from .notes import CONF_MASK
from .pipeline import (STAGE_ORDER, PipelineError, SongInput, read_warnings,
run_pipeline, song_slug)
from .ultrastar import get_header, parse_ultrastar
@@ -504,6 +506,26 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
return {"lines": lines, "missing_spans": missing_spans, "media": media,
"stale": stale, "grid": _chart_grid(work)}
@app.get("/api/jobs/{job_id}/pitch")
def get_pitch(job_id: str):
"""The detected vocal pitch, to draw behind the notes: UltraStar
semitones (0 = C4) every `step` seconds, null where nothing is sung."""
job = _finished_job(job_id)
path = cfg.work_dir / song_slug(job.inp) / "pitch.npz"
if not path.exists():
raise HTTPException(404)
data = np.load(path)
times, f0, conf = data["times"], data["f0"], data["conf"]
# every other 10ms frame is still finer than a note can be drawn
stride = 2
f0, conf = f0[::stride], conf[::stride]
voiced = (conf >= CONF_MASK) & (f0 > 0)
semis = 9 + 12 * np.log2(np.where(voiced, f0, 440.0) / 440.0)
hop = float(times[1] - times[0]) if len(times) > 1 else 0.01
return {"start": float(times[0]) if len(times) else 0.0, "step": hop * stride,
"pitch": [round(float(v), 2) if ok else None
for v, ok in zip(semis, voiced)]}
def _chart_grid(work: Path) -> dict | None:
"""The beat grid the chart was built on, for snapping edits to it."""
txt = work / "song.txt"
+47
View File
@@ -0,0 +1,47 @@
import assert from "node:assert/strict";
import test from "node:test";
import { beatLines, isBlackKey, lineSpan, noteName, pitchRange }
from "../../src/songtrace/static/sheet.js";
test("note names follow UltraStar's C4 = 0", () => {
assert.equal(noteName(0), "C4");
assert.equal(noteName(9), "A4");
assert.equal(noteName(-1), "B3");
assert.equal(noteName(13), "C♯5");
assert.equal(noteName(-13), "B2");
});
test("black keys are the sharps, in any octave", () => {
assert.deepEqual([0, 1, 2, 3, 4, 5, 6].map(isBlackKey),
[false, true, false, true, false, false, true]);
assert.equal(isBlackKey(-11), true); // C#3
assert.equal(isBlackKey(-12), false); // C3
});
test("the pitch range pads the line's notes and keeps a minimum height", () => {
assert.deepEqual(pitchRange([0, 5]), { lo: -4, hi: 10 }); // 15 rows
assert.deepEqual(pitchRange([0, 20]), { lo: -3, hi: 23 }); // padded only
assert.deepEqual(pitchRange([]), { lo: -7, hi: 7 });
});
test("beat lines sit on the chart's grid with bar weights", () => {
// 300 BPM: an UltraStar beat is 60 / (4 * 300) = 0.05s, from GAP 1.0s
const lines = beatLines(1.0, 1.85, { bpm: 300, gap_ms: 1000 });
assert.equal(lines.length, 18);
assert.deepEqual(lines.slice(0, 5).map(b => b.weight), [2, 0, 0, 0, 1]);
assert.equal(lines[16].weight, 2);
assert.ok(Math.abs(lines[4].t - 1.2) < 1e-9);
});
test("beats before GAP keep their weights", () => {
const lines = beatLines(0.75, 1.0, { bpm: 300, gap_ms: 1000 });
assert.deepEqual(lines.map(b => [b.beat, b.weight]),
[[-5, 0], [-4, 1], [-3, 0], [-2, 0], [-1, 0], [0, 2]]);
});
test("a line spans its notes and words", () => {
assert.deepEqual(lineSpan({ notes: [{ start: 2, end: 3 }], words: [{ start: 1.5, end: 2.5 }] }),
{ start: 1.5, end: 3 });
assert.equal(lineSpan({ notes: [], words: [] }), null);
});
+15
View File
@@ -0,0 +1,15 @@
import shutil
import subprocess
from pathlib import Path
import pytest
def test_static_js_unit_tests():
"""The page's own logic, run under Node's test runner."""
node = shutil.which("node")
if node is None:
pytest.skip("node is not installed")
tests = sorted(str(p) for p in (Path(__file__).parent / "js").glob("*.test.mjs"))
proc = subprocess.run([node, "--test", *tests], capture_output=True, text=True)
assert proc.returncode == 0, proc.stdout + proc.stderr
+26
View File
@@ -1977,3 +1977,29 @@ def test_a_stale_edit_can_be_discarded(tmp_path):
assert resp.status_code == 200 and resp.json()["stale"] == []
assert not (work / edits.EDITS_FILE).exists()
assert client.delete(f"/api/jobs/{second}/stale/0").status_code == 404
def test_pitch_curve_is_in_note_semitones_with_gaps_where_unvoiced(tmp_path):
import numpy as np
def runner(inp, cfg, log=print, force_from=None, stages=None):
result = chart_runner(inp, cfg)
times = np.arange(0, 0.1, 0.01)
f0 = np.array([440.0] * 4 + [261.63] * 4 + [0.0] * 2) # A4, C4, nothing
conf = np.array([0.9] * 4 + [0.9, 0.9, 0.1, 0.1] + [0.9] * 2)
np.savez(cfg.work_dir / song_slug(inp) / "pitch.npz", times=times, f0=f0, conf=conf)
return result
client, _ = make_client(tmp_path, runner)
job_id = make_charted_job(client)
body = client.get(f"/api/jobs/{job_id}/pitch").json()
assert body["start"] == 0.0 and body["step"] == pytest.approx(0.02)
assert body["pitch"][:2] == [9.0, 9.0] # A4 is 9 above C4
assert body["pitch"][2] == pytest.approx(0.0, abs=0.01) # C4
assert body["pitch"][3:] == [None, None] # unsure, then silent
def test_pitch_curve_is_404_without_a_pitch_track(tmp_path):
client, _ = make_client(tmp_path, chart_runner)
job_id = make_charted_job(client)
assert client.get(f"/api/jobs/{job_id}/pitch").status_code == 404