webui: move line breaks on the sheet

Line breaks were always placed by songtrace, midway between lines and
recomputed around edited ones, so a break could not be put where a singer
breathes. The sheet now draws the breaks either side of a line as markers
that drag along the beat grid, kept between the lines they separate, and
double-click back to automatic. Break moves share the line's undo history
and SAVE LINE, which saves the notes only if they changed, so moving a
break never locks either line's notes.

Moved breaks are stored in edits.json keyed by the line each comes before,
follow their lines when the lyrics change, and win over the computed break
on every rebuild, clamped so a break never lands inside a line's notes.
The review lists each break with that line and whether it was moved, and
PUT /api/jobs/{id}/breaks takes {line: seconds | null}. merge now passes
untouched notes through as they are rather than rebuilding them, which
added float noise to their durations.
This commit is contained in:
2026-10-13 16:24:17 -07:00
parent bdfd7d8429
commit 30e4f8f202
9 changed files with 258 additions and 23 deletions
+5
View File
@@ -70,6 +70,11 @@ next note, S splits, M merges with the next note, G/F/R make a note golden,
freestyle or rap, Delete removes, Ctrl+Z undoes, Ctrl+S saves and P plays
the line.
The ↵ markers either side of a line are its line breaks: drag one to move it
(it snaps to beats and stays between the lines it separates) and double-click
one to put it back where songtrace places it. Moving a break does not mark
either line's notes as edited, so they still follow a rebuild.
The **GRID** bar above the notes sets the chart's BPM and GAP. Changes redraw
the grid as you type and are charted on **APPLY**; ×2 and ÷2 fix a tempo
detected at double or half speed, and **ALIGN TO NOTE** moves GAP the least
+48 -9
View File
@@ -19,8 +19,9 @@ KINDS = {":", "*", "F", "R", "G"}
def _empty(lyrics: list[list[str]]) -> dict:
# grid: the BPM and GAP set in the editor, or None for the pipeline's own
return {"lyrics": lyrics, "lines": {}, "stale": [], "grid": None}
# grid: the BPM and GAP set in the editor, or None for the pipeline's own;
# breaks: line breaks moved by hand, keyed by the line each comes before
return {"lyrics": lyrics, "lines": {}, "stale": [], "grid": None, "breaks": {}}
def load(work: Path, lyrics: list[list[str]]) -> dict:
@@ -43,7 +44,7 @@ def load(work: Path, lyrics: list[list[str]]) -> dict:
def save(work: Path, data: dict) -> None:
path = work / EDITS_FILE
if not data["lines"] and not data["stale"] and not data.get("grid"):
if not (data["lines"] or data["stale"] or data.get("grid") or data.get("breaks")):
path.unlink(missing_ok=True)
return
tmp = path.with_suffix(".tmp")
@@ -66,7 +67,12 @@ def _remap(data: dict, lyrics: list[list[str]]) -> dict:
stale.append(edit)
else:
lines[str(to)] = {**edit, "line_index": to}
return {"lyrics": lyrics, "lines": lines, "stale": stale, "grid": data.get("grid")}
# a moved break goes with the line it comes before; with that line gone,
# there is nothing left for it to separate
breaks = {str(moved[int(k)]): t for k, t in data.get("breaks", {}).items()
if int(k) in moved}
return {"lyrics": lyrics, "lines": lines, "stale": stale, "grid": data.get("grid"),
"breaks": breaks}
def validate_notes(notes: list[dict]) -> list[dict]:
@@ -99,6 +105,30 @@ def clear_line(data: dict, index: int) -> bool:
return data["lines"].pop(str(index), None) is not None
def set_breaks(data: dict, changes: dict) -> None:
"""Move line breaks, each keyed by the line it comes before, to a time in
seconds; None puts one back to where the notes alone would place it."""
breaks = dict(data.get("breaks", {}))
for key, t in changes.items():
try:
line = int(key)
except (TypeError, ValueError):
raise ValueError(f"a break is keyed by the line it comes before, not {key!r}")
if not 0 < line < len(data["lyrics"]):
raise ValueError(f"no line {line} for a break to come before")
if t is None:
breaks.pop(str(line), None)
continue
try:
t = float(t)
except (TypeError, ValueError):
raise ValueError(f"a break needs a time in seconds, not {t!r}")
if not (math.isfinite(t) and t >= 0):
raise ValueError(f"a break needs a time of 0 or more, not {t}")
breaks[str(line)] = t
data["breaks"] = breaks
def set_grid(data: dict, bpm, gap_ms) -> None:
"""Set the chart's BPM (UltraStar's, a quarter of a beat per beat) and GAP."""
try:
@@ -125,10 +155,12 @@ def merge(notes: list[dict], breaks: list[float], data: dict
An edited line replaces the generated notes of that line, and the user's
timing wins: generated notes of earlier lines are cut back to end where it
starts, and those of later lines to start where it ends. Line breaks next
to an edited line move to the middle of the gap it leaves.
to an edited line move to the middle of the gap it leaves; a break moved
by hand stays where it was put, kept between the lines it separates.
"""
edited = {int(k): e["notes"] for k, e in data["lines"].items()}
if not edited:
moved = {int(k): t for k, t in data.get("breaks", {}).items()}
if not edited and not moved:
return notes, breaks
spans = {li: (ns[0]["start"], ns[-1]["start"] + ns[-1]["duration"])
for li, ns in edited.items() if ns}
@@ -143,7 +175,9 @@ def merge(notes: list[dict], breaks: list[float], data: dict
end = min(end, a)
elif li > eli:
start = max(start, b)
if end - start >= MIN_NOTE_SEC:
if (start, end) == (n["start"], n["start"] + n["duration"]):
kept.append(n)
elif end - start >= MIN_NOTE_SEC:
kept.append({**n, "start": start, "duration": end - start})
for li, ns in edited.items():
kept += [{**n, "line_index": li, "low_confidence": False} for n in ns]
@@ -160,13 +194,17 @@ def merge(notes: list[dict], breaks: list[float], data: dict
order = sorted(first, key=lambda li: first[li]["start"])
out_breaks = []
for prev, nxt in zip(order, order[1:]):
end = last[prev]["start"] + last[prev]["duration"]
if nxt in moved:
# where it was put, but never inside either line's notes
out_breaks.append(min(max(moved[nxt], end), max(first[nxt]["start"], end)))
continue
if prev not in edited and nxt not in edited:
# the generated break between these two lines, untouched
gen = [t for t in breaks if line_after(t) == nxt]
if gen:
out_breaks.append(gen[0])
continue
end = last[prev]["start"] + last[prev]["duration"]
out_breaks.append((end + first[nxt]["start"]) / 2)
return kept, out_breaks
@@ -189,6 +227,7 @@ def finalize(work: Path, lyrics: list[list[str]]) -> dict:
" the lyrics; set aside in the editor")
final = {"notes": notes, "breaks": breaks, "warnings": warnings,
"edited_lines": sorted(int(k) for k in data["lines"]),
"stale": data["stale"], "grid": data.get("grid")}
"stale": data["stale"], "grid": data.get("grid"),
"moved_breaks": sorted(int(k) for k in data.get("breaks", {}))}
(work / FINAL_FILE).write_text(json.dumps(final))
return final
+8
View File
@@ -148,3 +148,11 @@ export function history(initial) {
},
};
}
// Where a dragged line break lands: on a beat, but kept between the end of
// the line before it and the start of the line it introduces.
export function placeBreak(t, lo, hi, grid) {
const snapped = snap(t, grid);
if (hi < lo) return lo;
return Math.min(Math.max(snapped, lo), hi);
}
+5
View File
@@ -522,6 +522,11 @@ button { cursor: pointer; }
.sheet-beat { stroke: var(--border); stroke-width: 1; opacity: 0.35; }
.sheet-beat-1 { stroke: var(--control); opacity: 0.55; }
.sheet-beat-2 { stroke: var(--text-muted); opacity: 0.8; }
.sheet-breaks { pointer-events: none; }
.sheet-break { stroke: var(--text-muted); stroke-width: 1.5; stroke-dasharray: 5 4; }
.sheet-break-moved { stroke: var(--aqua); stroke-dasharray: none; }
.sheet-break-label { fill: var(--text-muted); stroke: none; font: 12px var(--font-mono); }
.sheet-break-label.sheet-break-moved { fill: var(--aqua); stroke: none; }
.sheet-curve { pointer-events: none; }
.sheet-curve polyline { fill: none; stroke: var(--aqua); stroke-width: 1.5; opacity: 0.9; }
.sheet-note { fill: var(--purple); }
+109 -13
View File
@@ -41,6 +41,9 @@ export const SHORTCUTS = [
["F", "freestyle"],
["R", "rap"],
["G / F / R", "again: back to normal"]]],
["Line break", [
["drag a ↵ marker", "move it, between the lines it separates"],
["double-click a ↵", "back to where songtrace puts it"]]],
["Syllable", [
["double-click", "edit, on a note"],
["Enter", "edit the selected note"]]],
@@ -264,12 +267,16 @@ export function createSheet(root, { seek, playLine, onSaved }) {
let hist = null; // the current line's notes, with undo
let baseline = null; // what the line holds on the server
let preview = null; // notes mid-drag, not yet in the history
let breakPreview = null; // a break mid-drag: { which, t }
let selected = new Set();
let drag = null;
let saving = false;
const visible = () => review.lines.filter(l => !l.dropped && lineSpan(l));
const notes = () => preview ?? hist?.present ?? [];
// the line being edited: its notes and the breaks either side of it, as
// one state so undo covers both. A break is its time, undefined where there
// is none to move, or null when it is to go back to automatic.
const notes = () => preview ?? hist?.present.notes ?? [];
const dirty = () => !!hist && hist.present !== baseline;
let gridDraft = null; // BPM and GAP being tried out, not yet applied
// the grid being drawn and snapped to: the one being tried out, if any
@@ -291,6 +298,12 @@ export function createSheet(root, { seek, playLine, onSaved }) {
const span = ts.length ? { start: Math.min(...ts), end: Math.max(...ts) } : lineSpan(line);
const range = pitchRange(notes().map(n => n.pitch));
const want = { t0: span.start - CONTEXT, t1: span.end + CONTEXT, ...range };
// the breaks either side are part of the line's page, so they stay on it
for (const slot of breakSlots()) {
const t = breakTime(slot);
want.t0 = Math.min(want.t0, t - 0.3);
want.t1 = Math.max(want.t1, t + 0.3);
}
layout = reset || !layout ? want : {
t0: Math.min(layout.t0, want.t0), t1: Math.max(layout.t1, want.t1),
lo: Math.min(layout.lo, want.lo), hi: Math.max(layout.hi, want.hi),
@@ -326,8 +339,16 @@ export function createSheet(root, { seek, playLine, onSaved }) {
function open(i) {
index = i;
const line = review.lines[i];
baseline = edit.withIds(line.notes.map(n => ({ ...n, kind: n.kind ?? ":" })));
const lines = visible();
const at = lines.findIndex(l => l.index === i);
const breakBefore = (l) => review.breaks?.find(b => b.line === l?.index)?.time;
baseline = {
notes: edit.withIds(line.notes.map(n => ({ ...n, kind: n.kind ?? ":" }))),
before: at > 0 ? breakBefore(line) : undefined,
after: breakBefore(lines[at + 1]),
};
hist = edit.history(baseline);
breakPreview = null;
preview = null;
selected = new Set();
say("");
@@ -351,12 +372,47 @@ export function createSheet(root, { seek, playLine, onSaved }) {
}
function change(next, keep = selected) {
hist.push(next);
hist.push({ ...hist.present, notes: next });
selected = new Set([...keep].filter(id => next.some(n => n.id === id)));
fit(false);
render();
}
function changeBreak(which, t) {
if (hist.present[which] === t) return;
hist.push({ ...hist.present, [which]: t });
render();
}
// the breaks either side of the line, each with the gap it may move in:
// from the end of the line before it to the start of the line after
function breakSlots() {
const ns = notes();
if (!hist || !ns.length) return [];
const lines = visible();
const at = lines.findIndex(l => l.index === index);
const prev = lines[at - 1];
const next = lines[at + 1];
const slots = [];
if (prev?.notes.length && hist.present.before !== undefined) {
slots.push({ which: "before", line: index,
lo: Math.max(...prev.notes.map(n => n.end)), hi: ns[0].start });
}
if (next?.notes.length && hist.present.after !== undefined) {
slots.push({ which: "after", line: next.index,
lo: Math.max(...ns.map(n => n.end)), hi: Math.min(...next.notes.map(n => n.start)) });
}
return slots;
}
// where a break is drawn; one going back to automatic is shown mid-gap,
// roughly where it will land
function breakTime(slot) {
const v = breakPreview?.which === slot.which ? breakPreview.t : hist.present[slot.which];
const t = v === null ? (slot.lo + slot.hi) / 2 : v;
return Math.min(Math.max(t, slot.lo), Math.max(slot.hi, slot.lo));
}
const undo = () => { if (hist?.canUndo) { hist.undo(); fit(false); render(); } };
const redo = () => { if (hist?.canRedo) { hist.redo(); fit(false); render(); } };
const one = () => (selected.size === 1 ? [...selected][0] : null);
@@ -420,9 +476,20 @@ export function createSheet(root, { seek, playLine, onSaved }) {
}
}
function save() {
// only what changed: a moved break alone must not mark the line edited
async function save() {
if (!dirty() || saving) return;
send(`lines/${index}`, "PUT", { notes: edit.toSaved(notes()) }, "saving…", "saved");
const state = hist.present;
const lines = visible();
const next = lines[lines.findIndex(l => l.index === index) + 1];
const moved = {};
if (state.before !== baseline.before) moved[index] = state.before;
if (state.after !== baseline.after && next) moved[next.index] = state.after;
if (state.notes !== baseline.notes &&
!await send(`lines/${index}`, "PUT", { notes: edit.toSaved(state.notes) }, "saving…", "saved")) {
return;
}
if (Object.keys(moved).length) await send("breaks", "PUT", moved, "saving…", "saved");
}
function revert() {
@@ -548,6 +615,18 @@ export function createSheet(root, { seek, playLine, onSaved }) {
}
for (const n of notes()) drawNote(n, true);
const marks = el("g", { class: "sheet-breaks" }, svg);
for (const slot of breakSlots()) {
const t = breakTime(slot);
const value = hist.present[slot.which];
const byHand = value !== null &&
(value !== baseline[slot.which] || review.breaks?.find(b => b.line === slot.line)?.moved);
const cls = `sheet-break${byHand ? " sheet-break-moved" : ""}`;
el("line", { x1: x(t), x2: x(t), y1: 0, y2: PLOT_HEIGHT, class: cls }, marks);
el("text", { x: x(t) + 3, y: 11, class: `${cls} sheet-break-label` }, marks)
.textContent = "↵";
}
syllables.replaceChildren(...notes().map(n => {
const s = div("sheet-syllable" + (selected.has(n.id) ? " sheet-syllable-selected" : ""),
n.text.trim() || "~");
@@ -628,6 +707,10 @@ export function createSheet(root, { seek, playLine, onSaved }) {
const py = e.clientY - box.top;
// by geometry, not the event's target: a render replaces the elements
// under the pointer, and the topmost note drawn wins
if (canEdit()) {
const brk = breakSlots().find(s => Math.abs(px - view.x(breakTime(s))) <= 5);
if (brk) return { px, py, note: null, brk, mode: "break" };
}
const pitch = view.pitch(py);
const note = [...notes()].reverse().find(n => n.pitch === pitch &&
px >= view.x(n.start) - 1 && px <= view.x(n.end) + 1) ?? null;
@@ -638,14 +721,15 @@ export function createSheet(root, { seek, playLine, onSaved }) {
mode = right - left > 3 * EDGE && px >= right - EDGE ? "end"
: right - left > 3 * EDGE && px <= left + EDGE ? "start" : "move";
}
return { px, py, note, mode };
return { px, py, note, brk: null, mode };
}
plot.addEventListener("pointermove", (e) => {
if (!view) return;
if (!drag) {
const { note, mode } = hit(e);
plot.style.cursor = !note || !canEdit() ? "crosshair"
const { note, brk, mode } = hit(e);
plot.style.cursor = brk ? "col-resize"
: !note || !canEdit() ? "crosshair"
: mode === "move" ? "grab" : "ew-resize";
return;
}
@@ -653,6 +737,12 @@ export function createSheet(root, { seek, playLine, onSaved }) {
const px = e.clientX - box.left;
const py = e.clientY - box.top;
if (Math.abs(px - drag.px) + Math.abs(py - drag.py) > 3) drag.moved = true;
if (drag.moved && drag.brk) {
const { which, lo, hi } = drag.brk;
breakPreview = { which, t: edit.placeBreak(view.time(px), lo, hi, grid()) };
render();
return;
}
if (!drag.moved || !drag.note) return;
if (drag.mode === "move") {
const spbPx = view.x(60 / (4 * grid().bpm)) - view.x(0);
@@ -671,8 +761,9 @@ export function createSheet(root, { seek, playLine, onSaved }) {
// sheet, and with it the keyboard
e.preventDefault();
plot.focus();
const { px, py, note, mode } = hit(e);
const { px, py, note, brk, mode } = hit(e);
if (px < GUTTER) return;
if (brk) plot.setPointerCapture(e.pointerId);
if (note && canEdit()) {
if (e.shiftKey) {
selected = new Set(selected);
@@ -684,18 +775,22 @@ export function createSheet(root, { seek, playLine, onSaved }) {
plot.style.cursor = mode === "move" ? "grabbing" : "ew-resize";
paintSelection();
}
drag = { px, py, note, mode, base: notes(), moved: false };
drag = { px, py, note, brk, mode, base: notes(), moved: false };
});
const endDrag = (e, cancelled) => {
if (!drag) return;
const d = drag;
drag = null;
if (preview) {
if (breakPreview) {
const { which, t } = breakPreview;
breakPreview = null;
if (cancelled) render(); else changeBreak(which, t);
} else if (preview) {
const next = preview;
preview = null;
if (cancelled) render(); else change(next);
} else if (!d.moved && !d.note && !cancelled) {
} else if (!d.moved && !d.note && !d.brk && !cancelled) {
selected = new Set();
render();
seek(view.time(d.px));
@@ -707,8 +802,9 @@ export function createSheet(root, { seek, playLine, onSaved }) {
plot.addEventListener("dblclick", (e) => {
if (!view || !canEdit()) return;
const { px, py, note } = hit(e);
const { px, py, note, brk } = hit(e);
if (px < GUTTER) return;
if (brk) { changeBreak(brk.which, null); return; } // back to automatic
if (note) { editText(note.id); return; }
const out = edit.insert(notes(), view.time(px), view.pitch(py), grid());
change(out.notes, new Set([out.id]));
+16 -1
View File
@@ -503,8 +503,17 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
for e in notes_data.get("stale", [])]
media = {kind: _resolve_media(job, header) is not None
for kind, header in MEDIA_HEADERS.items()}
# each break with the line it comes before, the key a move is saved by
moved = set(notes_data.get("moved_breaks", []))
starts = sorted((n["start"], n["line_index"]) for n in notes_data["notes"]
if "line_index" in n)
breaks = []
for t in notes_data.get("breaks", []):
after = next((li for start, li in starts if start >= t), None)
if after is not None:
breaks.append({"line": after, "time": t, "moved": after in moved})
return {"lines": lines, "missing_spans": missing_spans, "media": media,
"stale": stale, "grid": _chart_grid(work),
"stale": stale, "breaks": breaks, "grid": _chart_grid(work),
"grid_edited": bool(notes_data.get("grid")), "auto_bpm": _auto_bpm(work)}
@app.get("/api/jobs/{job_id}/pitch")
@@ -586,6 +595,12 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
g = _chart_grid(work)
return g["bpm"] if g else None
@app.put("/api/jobs/{job_id}/breaks")
def save_breaks(job_id: str, body: dict = Body(...)):
"""Move line breaks: {line: seconds}, keyed by the line each comes
before; null puts one back where the notes alone would place it."""
return _edit(job_id, lambda data, lyrics: edits.set_breaks(data, body))
@app.put("/api/jobs/{job_id}/grid")
def save_grid(job_id: str, body: dict = Body(...)):
"""Set the chart's BPM and GAP, kept through rebuilds like any edit."""
+7
View File
@@ -120,3 +120,10 @@ test("history undoes and redoes, and a new change drops the redo", () => {
assert.equal(h.canRedo, false);
assert.equal(h.present, "d");
});
test("a dragged break snaps to a beat but stays between its lines", () => {
assert.ok(near(m.placeBreak(1.52, 1.4, 1.8, GRID), 1.5));
assert.equal(m.placeBreak(1.0, 1.42, 1.8, GRID), 1.42); // not into the line before
assert.equal(m.placeBreak(2.5, 1.4, 1.78, GRID), 1.78); // nor into the line after
assert.equal(m.placeBreak(1.6, 1.9, 1.8, GRID), 1.9); // lines overlap: at the end of the first
});
+35
View File
@@ -152,3 +152,38 @@ def test_finalize_reports_the_grid(tmp_path):
edits.set_grid(data, 300, 1000)
edits.save(tmp_path, data)
assert edits.finalize(tmp_path, LYRICS)["grid"] == {"bpm": 300.0, "gap_ms": 1000.0}
def test_moved_breaks_are_checked():
data = edits._empty(LYRICS)
edits.set_breaks(data, {"1": 3.5, 2: "6.25"})
assert data["breaks"] == {"1": 3.5, "2": 6.25}
edits.set_breaks(data, {1: None})
assert data["breaks"] == {"2": 6.25}
for bad in ({0: 1.0}, {3: 1.0}, {"x": 1.0}, {1: -1}, {1: "soon"}, {1: float("inf")}):
with pytest.raises(ValueError):
edits.set_breaks(data, bad)
def test_a_moved_break_stays_put_between_its_lines():
data = edits._empty(LYRICS)
edits.set_breaks(data, {1: 2.5})
notes, breaks = edits.merge(GENERATED, BREAKS, data)
assert notes == GENERATED # moving a break locks no notes
assert breaks == [2.5, 6.0]
def test_a_moved_break_never_lands_inside_a_line():
data = edits._empty(LYRICS)
edits.set_breaks(data, {1: 1.0, 2: 9.0}) # into line 0's notes, past line 2's start
_, breaks = edits.merge(GENERATED, BREAKS, data)
assert breaks == [pytest.approx(1.9), 8.0]
def test_moved_breaks_follow_their_lines_and_keep_the_file(tmp_path):
data = edits._empty(LYRICS)
edits.set_breaks(data, {1: 3.5, 2: 6.5})
edits.save(tmp_path, data)
assert (tmp_path / edits.EDITS_FILE).exists()
moved = edits.load(tmp_path, [["zero"], LYRICS[0], LYRICS[1], ["other"]])
assert moved["breaks"] == {"2": 3.5} # line 2's words changed: its break goes
+25
View File
@@ -2049,3 +2049,28 @@ def test_reset_on_a_song_built_before_tempo_was_recorded_restores_its_bpm(tmp_pa
client.put(f"/api/jobs/{job_id}/grid", json={"bpm": 240, "gap_ms": 900})
body = client.delete(f"/api/jobs/{job_id}/grid").json()
assert body["grid"]["bpm"] == 300.0 and body["auto_bpm"] == 300.0
def test_moving_a_break_recharts_and_is_reported(tmp_path):
from songtrace.ultrastar import LineBreak, parse_ultrastar
client, cfg = make_client(tmp_path, chart_runner)
job_id = make_charted_job(client)
assert client.get(f"/api/jobs/{job_id}/review").json()["breaks"] == [
{"line": 1, "time": 2.0, "moved": False}]
resp = client.put(f"/api/jobs/{job_id}/breaks", json={"1": 2.8})
assert resp.status_code == 200
assert resp.json()["breaks"] == [{"line": 1, "time": 2.8, "moved": True}]
assert not any(line["edited"] for line in resp.json()["lines"])
packaged = cfg.output_dir / "A - T" / "A - T.txt"
body = parse_ultrastar(packaged.read_text(encoding="utf-8")).body
# 300 BPM: 0.05s beats from GAP 1.0s, so 2.8s is beat 36
assert [b.beat for b in body if isinstance(b, LineBreak)] == [36]
back = client.put(f"/api/jobs/{job_id}/breaks", json={"1": None}).json()
assert back["breaks"] == [{"line": 1, "time": 2.0, "moved": False}]
def test_a_bad_break_is_refused(tmp_path):
client, _ = make_client(tmp_path, chart_runner)
job_id = make_charted_job(client)
for body in ({"0": 1.0}, {"9": 1.0}, {"1": -2}, {"1": "x"}):
assert client.put(f"/api/jobs/{job_id}/breaks", json=body).status_code == 400