notes: judge low confidence against the song, and flag unsingable words

A fixed 0.5 alignment-score cutoff flagged 73-99% of notes on sung
vocals, so the flag pointed nowhere. Scores still separate bad words from
good ones, but their scale differs from song to song: flag a word below a
quarter of the song's median score, or one shorter than 0.08s or longer
than 3s, which no singer produces and which a word stretched over
something else does. Flagged notes less than 2s apart share one warning.

.gitea/workflows/tests.yml

@@ -70,6 +70,9 @@ jobs:
           apt-get update
           apt-get install -y --no-install-recommends python3 python3-venv
           python3 -m venv /opt/venv
+          # bookworm's pip 23.0 rejects the PyTorch index's typing_extensions
+          # wheel over its name's spelling, then cannot build it from source
+          /opt/venv/bin/pip install --retries 5 --upgrade pip
           echo /opt/venv/bin >> "$GITHUB_PATH"
 
       # The CPU build first: torch from PyPI is the CUDA build, several GB of

README.md

@@ -177,6 +177,12 @@ to redo a stage anyway — pick `package` to repackage without re-downloading.
 It can only widen what is rerun, so an edit that invalidates an earlier stage
 still wins.
 
+**REBUILD ALL FROM…**, under CREATE SONG, does the same for every song at once:
+pick the stage an update improved and each song is rebuilt from there with the
+inputs it has now. Every song starts ticked; untick any already rebuilt by hand
+(each shows when it was last built). Songs still in progress are skipped, as
+are songs whose uploaded video is gone when the stage is `acquire`.
+
 ## YouTube download failures
 
 `HTTP Error 403: Forbidden` from the acquire stage means YouTube served the

src/songtrace/notes.py

@@ -7,7 +7,17 @@ import pyphen
 from .languages import LANGUAGES
 
 CONF_MASK = 0.5          # pitch frames below this are unvoiced
-LOW_SCORE = 0.5          # word alignment score below this -> low_confidence
+# A word is low-confidence when it scores below this share of the song's
+# median alignment score: the scale differs a lot from song to song, so no
+# fixed cutoff holds across them.
+LOW_SCORE_OF_MEDIAN = 0.25
+# ...or when no one sings a word that fast, or it was stretched over more than
+# it can be holding.
+MIN_WORD_SEC = 0.08
+MAX_WORD_SEC = 3.0
+# Flagged notes closer than this share one warning, so the list reads as
+# places to look rather than one line per note.
+WARN_GAP_SEC = 2.0
 SPLIT_SEMITONES = 0.8
 MIN_SEG_SEC = 0.05
 MIN_NOTE_SEC = 0.05      # alignment may hand us zero-length words
@@ -84,6 +94,7 @@ def build_notes(rows: list[dict], times, f0, conf, language: str | None,
                 ) -> tuple[list[TimedNote], list[float], list[str]]:
     notes: list[TimedNote] = []
     breaks: list[float] = []
+    floor = LOW_SCORE_OF_MEDIAN * float(np.median([r["score"] for r in rows])) if rows else 0.0
     prev_line, prev_end = None, 0.0
     for row in rows:
         if prev_line is not None and row["line_index"] > prev_line:
@@ -97,7 +108,8 @@ def build_notes(rows: list[dict], times, f0, conf, language: str | None,
                                _text(row["word"], 0, line_initial), True,
                                row["line_index"]))
             continue
-        low = row["score"] < LOW_SCORE
+        low = (row["score"] < floor
+               or not MIN_WORD_SEC <= row["end"] - row["start"] <= MAX_WORD_SEC)
         sylls = syllabify(row["word"], language)
         if len(segs) < len(sylls):
             segs = _subdivide(segs, len(sylls))
@@ -160,16 +172,19 @@ def _fmt(t: float) -> str:
 
 def _warnings(notes: list[TimedNote], dropped_lines=(), missing_spans=(),
              ref_lines=()) -> list[str]:
-    warnings, run = [], []
+    runs: list[list[TimedNote]] = []
-    for n in list(notes) + [None]:
+    for n in notes:
-        if n is not None and n.low_confidence:
+        if not n.low_confidence:
-            run.append(n)
+            continue
-        elif run:
+        last = runs[-1][-1] if runs else None
-            first, last = run[0], run[-1]
+        if last is not None and n.start - (last.start + last.duration) < WARN_GAP_SEC:
-            warnings.append(
+            runs[-1].append(n)
-                f"{len(run)} low-confidence note{'' if len(run) == 1 else 's'} "
+        else:
-                f"near {_fmt(first.start)}–{_fmt(last.start + last.duration)}")
+            runs.append([n])
-            run = []
+    warnings = [
+        f"{len(run)} low-confidence note{'' if len(run) == 1 else 's'} "
+        f"near {_fmt(run[0].start)}–{_fmt(run[-1].start + run[-1].duration)}"
+        for run in runs]
     for group in _consecutive_runs(dropped_lines):
         n = len(group)
         text = " ".join(ref_lines[group[0]]) if group[0] < len(ref_lines) else ""

src/songtrace/stages/align.py

@@ -1,12 +1,24 @@
 import json
 from pathlib import Path
 
+import numpy as np
+import soundfile as sf
+
 from ..lyrics import alignment_token, normalize_lyrics
 from ..models import on_release
 from ..pipeline import Ctx, PipelineError, Stage
-from .transcribe import classify_lyrics_match, load_transcript, lyrics_similarity, vocal_sections
+from ..whisper_client import TranscriptSegment
+from .transcribe import (classify_lyrics_match, load_transcript, lyrics_similarity,
+                         recovered_spans, vocal_sections)
 
 SIMILARITY_THRESHOLD = 0.4
+# MMS_FA's wildcard token: any audio the transcript does not cover
+STAR = "*"
+# Singing is audio within this much of the stem's loud level; separation
+# bleed sits far below it.
+SINGING_BELOW_LOUD_DB = 25.0
+# Shorter stretches are breaths, ad-libs or Whisper's loose segment edges.
+MIN_UNHEARD_SEC = 3.0
 
 # a stage aligns section by section, so the model is cached across the calls
 # within one run and dropped at the end of it
@@ -40,7 +52,7 @@ def force_align_section(vocals_path: Path, start: float, end: float,
         waveform[:, lo:hi], sr, bundle.sample_rate)
     # MMS_FA only models a-z and apostrophe; anything else becomes the star token.
     vocab = bundle.get_dict()
-    tokens = ["".join(c if c in vocab else "*" for c in t) for t in tokens]
+    tokens = ["".join(c if c in vocab else STAR for c in t) for t in tokens]
     with torch.inference_mode():
         emission, _ = _get_model(device)(waveform.to(device))
         spans = bundle.get_aligner()(emission[0], bundle.get_tokenizer()(tokens))
@@ -50,6 +62,39 @@ def force_align_section(vocals_path: Path, start: float, end: float,
              sum(t.score for t in s) / len(s)) for s in spans]
 
 
+def untranscribed_singing(vocals_path: Path, segments,
+                          frame: float = 0.05) -> list[tuple[float, float]]:
+    """Stretches of the vocal stem with singing in them but no transcript."""
+    data, sr = sf.read(str(vocals_path), dtype="float32", always_2d=True)
+    hop = int(sr * frame)
+    n = len(data) // hop
+    if n == 0:
+        return []
+    rms = np.sqrt((data[:n * hop].mean(1).reshape(n, hop) ** 2).mean(1))
+    db = 20 * np.log10(rms + 1e-10)
+    sung = db >= np.quantile(db, 0.9) - SINGING_BELOW_LOUD_DB
+    times = np.arange(n) * frame
+    for s in segments:
+        sung[(times >= min(s.start, s.end)) & (times < max(s.start, s.end))] = False
+    spans, start = [], None
+    for t, on in zip(times, sung):
+        if on and start is None:
+            start = float(t)
+        elif not on and start is not None:
+            spans.append((start, float(t)))
+            start = None
+    if start is not None:
+        spans.append((start, float(times[-1] + frame)))
+    # a breath or a quiet syllable must not split one unsung passage in two
+    merged: list[list[float]] = []
+    for a, b in spans:
+        if merged and a - merged[-1][1] < 1.0:
+            merged[-1][1] = b
+        else:
+            merged.append([a, b])
+    return [(a, b) for a, b in merged if b - a >= MIN_UNHEARD_SEC]
+
+
 def _assign_lines_to_sections(line_indices: list[int], sections) -> list[list[int]]:
     """Split matched lyric line indices across vocal sections proportionally to section duration."""
     total = sum(b - a for a, b in sections) or 1.0
@@ -92,9 +137,20 @@ def _enforce_monotonic(rows: list[dict]) -> None:
 def _align_section(vocals: Path, sec: tuple[float, float], words: list[tuple[str, int]],
                    language: str | None, device: str) -> list[dict]:
     tokens = [alignment_token(w, language) for w, _ in words]
-    alignable = [t for t in tokens if t]
+    # A star before, between and after lines lets speech, laughter or an
-    aligned = iter(force_align_section(vocals, sec[0], sec[1], alignable, device)
+    # ad-lib there go unexplained rather than claim the next line's letters.
-                   if alignable else [])
+    seq, prev_line = [], None
+    for (_, line_index), token in zip(words, tokens):
+        if not token:
+            continue
+        if line_index != prev_line:
+            seq.append(STAR)
+            prev_line = line_index
+        seq.append(token)
+    if seq:
+        seq.append(STAR)
+    spans = force_align_section(vocals, sec[0], sec[1], seq, device) if seq else []
+    aligned = iter(span for span, token in zip(spans, seq) if token != STAR)
     rows, blank = [], (None, None, 0.0)
     for (word, line_index), token in zip(words, tokens):
         start, end, score = next(aligned, blank) if token else blank
@@ -112,11 +168,18 @@ def _run(ctx: Ctx) -> None:
         raise PipelineError(
             f"lyrics don't match audio (similarity {sim:.2f}) — "
             "wrong lyrics or wrong recording?")
-    # normalised so an inverted span can never collapse a section onto one instant
-    sections = [(min(a, b), max(a, b)) for a, b in vocal_sections(segments)]
-    dropped_lines, missing_spans = classify_lyrics_match(segments, ref)
-    matched = [i for i in range(len(ref.lines)) if i not in set(dropped_lines)]
     vocals = ctx.work / "vocals.wav"
+    unheard = untranscribed_singing(vocals, segments)
+    dropped_lines, missing_spans = classify_lyrics_match(segments, ref,
+                                                         unheard=unheard)
+    # lines Whisper missed are aligned against the singing it had no text for;
+    # only that singing, so an untranscribed ad-lib cannot draw lines into it
+    recovered = [TranscriptSegment(a, b, "")
+                 for a, b in recovered_spans(segments, ref, unheard)]
+    # normalised so an inverted span can never collapse a section onto one instant
+    sections = [(min(a, b), max(a, b))
+                for a, b in vocal_sections(segments + recovered)]
+    matched = [i for i in range(len(ref.lines)) if i not in set(dropped_lines)]
     rows = []
     for sec, line_idxs in zip(sections, _assign_lines_to_sections(matched, sections)):
         words = [(w, li) for li in line_idxs for w in ref.lines[li]]

src/songtrace/stages/transcribe.py

@@ -39,28 +39,74 @@ def lyrics_similarity(segments, ref: RefLyrics) -> float:
     return matcher.ratio()
 
 
-def classify_lyrics_match(segments, ref: RefLyrics, pad: float = 1.0
+def _line_times(segments, matcher, seg_of, line_of) -> dict[int, tuple[float, float]]:
+    """For each reference line that was heard, the start of the first and the
+    end of the last transcript segment its words were heard in. Words in a
+    'replace' block are paired with heard words proportionally."""
+    times: dict[int, tuple[float, float]] = {}
+    for tag, ai1, ai2, bj1, bj2 in matcher.get_opcodes():
+        if tag not in ("equal", "replace"):
+            continue
+        for j in range(bj1, bj2):
+            seg = segments[seg_of[ai1 + (j - bj1) * (ai2 - ai1) // (bj2 - bj1)]]
+            a, b = times.get(line_of[j], (seg.start, seg.end))
+            times[line_of[j]] = (min(a, seg.start), max(b, seg.end))
+    return times
+
+
+def _unheard_runs(segments, ref: RefLyrics, unheard
+                  ) -> tuple[list[list[int]], list[list[tuple[float, float]]]]:
+    """Runs of reference lines with nothing heard for them, each with the
+    untranscribed singing between the heard lines either side of it."""
+    matcher, seg_of, line_of = match_lyrics(segments, ref)
+    times = _line_times(segments, matcher, seg_of, line_of)
+    alignable = set(line_of)
+    runs, spans, run, prev_end = [], [], [], 0.0
+
+    def close(next_start: float) -> None:
+        if run:
+            runs.append(list(run))
+            spans.append([(max(a, prev_end), min(b, next_start)) for a, b in unheard
+                          if a < next_start and b > prev_end])
+            run.clear()
+
+    for li in range(len(ref.lines)):
+        if li in times:
+            close(times[li][0])
+            prev_end = times[li][1]
+        elif li in alignable:
+            run.append(li)
+    close(float("inf"))
+    return runs, spans
+
+
+def recovered_spans(segments, ref: RefLyrics, unheard) -> list[tuple[float, float]]:
+    """The untranscribed singing that stands in for lines Whisper missed, so
+    alignment can place them there."""
+    return _merge_spans([sp for spans in _unheard_runs(segments, ref, unheard)[1]
+                         for sp in spans])
+
+
+def classify_lyrics_match(segments, ref: RefLyrics, pad: float = 1.0, unheard=()
                           ) -> tuple[list[int], list[tuple[float, float]]]:
     """Split reference lines into matched vs. dropped, and flag sung spans no
     reference line explains.
 
     dropped_lines: reference line indices covered only by 'insert' opcodes
-    (present in the lyrics, never heard). missing_spans: the padded, merged
+    (present in the lyrics, never heard), unless `unheard` (sung spans the
-    bounds of the heard segments behind 'delete' opcodes (heard, but matching
+    transcript has no text for) has singing between the heard lines either
-    no reference word). A segment can also hold matched lines, so a span
+    side of them: then Whisper missed them, not the singer.
-    marks where to look, not words to discard.
-    """
-    matcher, seg_of, line_of = match_lyrics(segments, ref)
-    covered = [False] * len(line_of)
-    for tag, _, _, bj1, bj2 in matcher.get_opcodes():
-        if tag in ("equal", "replace"):
-            for j in range(bj1, bj2):
-                covered[j] = True
-    by_line = {}
-    for j, li in enumerate(line_of):
-        by_line.setdefault(li, []).append(covered[j])
-    dropped_lines = sorted(li for li, cov in by_line.items() if not any(cov))
 
+    missing_spans: the padded, merged bounds of the heard segments behind
+    'delete' opcodes (heard, but matching no reference word). A segment can
+    also hold matched lines, so a span marks where to look, not words to
+    discard.
+    """
+    runs, run_spans = _unheard_runs(segments, ref, unheard)
+    dropped_lines = sorted(li for run, sp in zip(runs, run_spans) if not sp
+                           for li in run)
+
+    matcher, seg_of, _ = match_lyrics(segments, ref)
     spans = []
     for tag, ai1, ai2, _, _ in matcher.get_opcodes():
         if tag != "delete":

src/songtrace/static/index.html

@@ -113,6 +113,61 @@ body {
 #sidebar .eyebrow, #jobs .empty-note { padding: 0 18px; }
 #create-song { display: block; width: calc(100% - 36px); margin: 0 18px 20px; padding: 9px 12px; }
 
+#rebuild-all { display: block; margin: -8px 18px 20px; }
+#rebuild-all-note { margin: -12px 18px 20px; }
+
+#rebuild-all-dialog {
+  width: min(480px, calc(100vw - 32px));
+  padding: 26px 28px;
+  color: var(--text);
+  background: var(--panel);
+  border: 1px solid var(--border);
+}
+#rebuild-all-dialog::backdrop { background: rgb(10 10 15 / 0.75); }
+#rebuild-all-dialog h2 {
+  margin: 0 0 14px;
+  font-family: var(--font-display);
+  font-weight: 700;
+  font-size: 0.8rem;
+  letter-spacing: 0.22em;
+  color: var(--text-muted);
+}
+#rebuild-all-form { display: grid; gap: 16px; }
+#rebuild-all-form p { margin: 0; font-size: 0.88rem; line-height: 1.5; }
+#rebuild-all-count { color: var(--warn); }
+#rebuild-all-songs { margin: 0; padding: 0; border: 0; min-width: 0; }
+#rebuild-all-songs legend {
+  margin-bottom: 6px;
+  padding: 0;
+  font-family: var(--font-display);
+  font-weight: 600;
+  font-size: 0.68rem;
+  letter-spacing: 0.14em;
+  text-transform: uppercase;
+  color: var(--text-muted);
+}
+#rebuild-all-list {
+  max-height: 40vh;
+  overflow-y: auto;
+  border-top: 1px solid var(--border);
+  border-bottom: 1px solid var(--border);
+}
+.rebuild-all-row {
+  display: grid;
+  grid-template-columns: auto 1fr;
+  gap: 1px 10px;
+  align-items: baseline;
+  padding: 7px 4px;
+  font-size: 0.88rem;
+  cursor: pointer;
+}
+.rebuild-all-row input { accent-color: var(--purple); }
+.rebuild-all-row:has(input:disabled) { cursor: default; color: var(--text-muted); }
+.rebuild-all-row .meta { grid-column: 2; font-size: 0.76rem; color: var(--text-muted); }
+.rebuild-all-every { padding-top: 0; color: var(--text-muted); }
+#rebuild-all-error { min-height: 1.2em; color: var(--fail); }
+#rebuild-all-error:empty { display: none; }
+
 #jobs { display: flex; flex-direction: column; gap: 1px; margin-top: 8px; }
 
 .job {
@@ -518,11 +573,39 @@ button { cursor: pointer; }
   </div>
   <div>
     <button type="button" id="create-song" class="primary">CREATE SONG</button>
+    <button type="button" id="rebuild-all" class="action">REBUILD ALL FROM…</button>
+    <p id="rebuild-all-note" class="empty-note" role="status" hidden></p>
     <h2 class="eyebrow">QUEUE</h2>
     <div id="jobs"></div>
   </div>
 </aside>
 
+<dialog id="rebuild-all-dialog" aria-labelledby="rebuild-all-heading">
+  <form id="rebuild-all-form" method="dialog">
+    <h2 id="rebuild-all-heading">REBUILD ALL SONGS</h2>
+    <p>Reruns the songs ticked below from the chosen stage onwards with the
+      inputs they have now, for when a songtrace update improves that stage.</p>
+    <label class="field">
+      <span>Restart from</span>
+      <select name="force_from" required></select>
+    </label>
+    <fieldset id="rebuild-all-songs">
+      <legend>Songs</legend>
+      <label class="rebuild-all-row rebuild-all-every">
+        <input type="checkbox" id="rebuild-all-toggle" checked>
+        <span>All songs</span>
+      </label>
+      <div id="rebuild-all-list"></div>
+    </fieldset>
+    <p id="rebuild-all-count"></p>
+    <p id="rebuild-all-error" role="alert"></p>
+    <div class="form-actions">
+      <button type="submit" class="primary">REBUILD SELECTED</button>
+      <button type="button" class="action" id="rebuild-all-cancel">CANCEL</button>
+    </div>
+  </form>
+</dialog>
+
 <main id="main">
   <section id="new-song">
     <h2>NEW SONG</h2>
@@ -702,6 +785,95 @@ createSong.addEventListener("click", () => {
   form.elements.title.focus();
 });
 
+const rebuildAllButton = document.getElementById("rebuild-all");
+const rebuildAllNote = document.getElementById("rebuild-all-note");
+const rebuildAllDialog = document.getElementById("rebuild-all-dialog");
+const rebuildAllForm = document.getElementById("rebuild-all-form");
+const rebuildAllCount = document.getElementById("rebuild-all-count");
+const rebuildAllError = document.getElementById("rebuild-all-error");
+
+const rebuildAllList = document.getElementById("rebuild-all-list");
+const rebuildAllToggle = document.getElementById("rebuild-all-toggle");
+const rebuildAllSubmit = rebuildAllForm.querySelector("button[type=submit]");
+
+function rebuildAllBoxes() {
+  return [...rebuildAllList.querySelectorAll("input:not(:disabled)")];
+}
+
+function updateRebuildAllCount() {
+  const boxes = rebuildAllBoxes();
+  const chosen = boxes.filter(b => b.checked).length;
+  rebuildAllToggle.checked = chosen === boxes.length && chosen > 0;
+  rebuildAllToggle.indeterminate = chosen > 0 && chosen < boxes.length;
+  rebuildAllCount.textContent = chosen === 1 ? "1 song will be rebuilt."
+                                             : `${chosen} songs will be rebuilt.`;
+  // with nothing ticked no ids are sent, which the API reads as every song
+  rebuildAllSubmit.disabled = chosen === 0;
+}
+
+function rebuildAllRow(song) {
+  const busy = song.status === "queued" || song.status === "running";
+  const row = document.createElement("label");
+  row.className = "rebuild-all-row";
+  const box = document.createElement("input");
+  box.type = "checkbox";
+  box.name = "ids";
+  box.value = song.id;
+  box.checked = !busy;
+  box.disabled = busy;
+  const name = document.createElement("span");
+  name.textContent = `${song.title} · ${song.artist}`;
+  const meta = document.createElement("span");
+  meta.className = "meta";
+  const when = song.finished_at && new Date(song.finished_at * 1000).toLocaleString(
+    undefined, { dateStyle: "medium", timeStyle: "short" });
+  meta.textContent = busy ? "in progress, skipped"
+    : `${song.status === "failed" ? "failed" : song.rebuilt_from ? "rebuilt" : "built"} ${when || ""}`.trim();
+  row.append(box, name, meta);
+  return row;
+}
+
+rebuildAllButton.addEventListener("click", async () => {
+  rebuildAllError.textContent = "";
+  rebuildAllForm.reset();
+  const resp = await fetch("/api/rebuild-all");
+  const songs = resp.ok ? await resp.json() : [];
+  rebuildAllList.replaceChildren(...songs.map(rebuildAllRow));
+  updateRebuildAllCount();
+  rebuildAllDialog.showModal();
+});
+
+rebuildAllList.addEventListener("change", updateRebuildAllCount);
+rebuildAllToggle.addEventListener("change", () => {
+  for (const box of rebuildAllBoxes()) box.checked = rebuildAllToggle.checked;
+  updateRebuildAllCount();
+});
+
+document.getElementById("rebuild-all-cancel").addEventListener(
+  "click", () => rebuildAllDialog.close());
+
+rebuildAllForm.addEventListener("submit", async (e) => {
+  e.preventDefault();
+  const stage = rebuildAllForm.elements.force_from.value;
+  const resp = await fetch("/api/rebuild-all", {
+    method: "POST", body: new FormData(rebuildAllForm) });
+  if (!resp.ok) {
+    const body = await resp.json().catch(() => ({}));
+    rebuildAllError.textContent = body.detail || `error: ${resp.status}`;
+    return;
+  }
+  const { ids, skipped } = await resp.json();
+  rebuildAllDialog.close();
+  let note = `Queued ${ids.length} rebuild${ids.length === 1 ? "" : "s"} from ${stage}.`;
+  if (skipped.length) {
+    const reasons = [...new Set(skipped.map(s => s.reason))].join("; ");
+    note += ` Skipped ${skipped.length}: ${reasons}.`;
+  }
+  rebuildAllNote.textContent = note;
+  rebuildAllNote.hidden = false;
+  refreshJobs();
+});
+
 function jobCard(job) {
   const card = document.createElement("button");
   card.type = "button";
@@ -780,6 +952,8 @@ async function loadStages() {
   const select = form.elements.force_from;
   select.replaceChildren(new Option("only what changed", ""),
                          ...STAGES.map(s => new Option(s, s)));
+  rebuildAllForm.elements.force_from.replaceChildren(
+    new Option("choose a stage", ""), ...STAGES.map(s => new Option(s, s)));
 }
 
 function renderStageTrack(job) {

src/songtrace/web.py

@@ -329,6 +329,51 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
             rebuild_policy.plan(parent.inp, inp), force_from))
         return _enqueue(new_id, inp, rebuilt_from=job_id, force_stage=force_from)
 
+    def _current_songs() -> list[Job]:
+        """The job that stands for each song now, newest first."""
+        with lock:
+            # a job that was rebuilt is history: its rebuild is the song now,
+            # possibly under another slug
+            rebuilt = {j.rebuilt_from for j in jobs.values()}
+            latest = {song_slug(j.inp): j for j in (jobs[i] for i in order)
+                      if j.id not in rebuilt}
+            position = {job_id: n for n, job_id in enumerate(order)}
+            return sorted(latest.values(), key=lambda j: position[j.id], reverse=True)
+
+    @app.get("/api/rebuild-all")
+    def rebuild_all_candidates():
+        return [{"id": j.id, "title": j.title, "artist": j.artist,
+                 "status": j.status, "rebuilt_from": j.rebuilt_from,
+                 "finished_at": j.finished_at}
+                for j in _current_songs()]
+
+    @app.post("/api/rebuild-all", status_code=201)
+    def rebuild_all(force_from: str = Form(...), ids: list[str] | None = Form(None)):
+        """Rebuild every song, or the ones in `ids`, from `force_from` on, for
+        when songtrace itself improved that stage. Each is an ordinary rebuild
+        with unchanged inputs."""
+        if force_from not in STAGE_ORDER:
+            raise HTTPException(400, f"unknown stage {force_from!r}")
+        songs = _current_songs()
+        queued, skipped = [], []
+        if ids is not None:
+            current = {j.id for j in songs}
+            skipped = [{"id": i, "reason": "no longer the current job for its song"}
+                       for i in ids if i not in current]
+            songs = [j for j in songs if j.id in set(ids)]
+        for job in songs:
+            if job.status in ("queued", "running"):
+                skipped.append({"id": job.id, "reason": "still in progress"})
+                continue
+            try:
+                _require_source(job.inp, job.inp, force_from)
+            except HTTPException as e:
+                skipped.append({"id": job.id, "reason": e.detail})
+                continue
+            queued.append(_enqueue(uuid.uuid4().hex, job.inp, rebuilt_from=job.id,
+                                   force_stage=force_from)["id"])
+        return {"ids": queued, "skipped": skipped}
+
     @app.get("/api/stages")
     def list_stages():
         return {"stages": list(STAGE_ORDER)}

tests/test_align.py

@@ -1,11 +1,23 @@
 import json
 
+import numpy as np
 import pytest
+import soundfile as sf
 
 from songtrace.config import Config
 from songtrace.lyrics import normalize_lyrics
 from songtrace.pipeline import Ctx, PipelineError, SongInput
 from songtrace.stages import align
+from songtrace.whisper_client import TranscriptSegment
+
+REAL_UNTRANSCRIBED_SINGING = align.untranscribed_singing
+
+
+@pytest.fixture(autouse=True)
+def no_untranscribed_singing(monkeypatch):
+    """The tests have no vocals.wav; those about missed singing set their own."""
+    monkeypatch.setattr(align, "untranscribed_singing", lambda vocals, segments: [])
+
 
 LYRICS = "it's one two three\nfour five six seven eight"
 
@@ -160,3 +172,66 @@ def test_align_flags_missing_span_for_unmatched_singing(tmp_path, monkeypatch):
     meta = json.loads((tmp_path / "w" / "align_meta.json").read_text())
     assert meta["dropped_lines"] == []
     assert meta["missing_spans"] == [[19.0, 24.0]]
+
+
+def _tone(seconds, sr, level):
+    t = np.arange(int(seconds * sr)) / sr
+    return level * np.sin(2 * np.pi * 220 * t)
+
+
+def test_untranscribed_singing_finds_sung_audio_the_transcript_skips(
+        tmp_path):
+    sr = 16000
+    rng = np.random.default_rng(0)
+    audio = np.concatenate([
+        _tone(2, sr, 0.3),           # 0-2s: sung and transcribed
+        np.zeros(2 * sr),            # 2-4s: silence
+        _tone(5, sr, 0.3),           # 4-9s: sung, never transcribed
+        np.zeros(3 * sr),            # 9-12s: silence
+        _tone(1, sr, 0.3),           # 12-13s: too short to count
+        np.zeros(3 * sr),
+    ]) + 0.001 * rng.standard_normal(16 * sr)   # separation bleed
+    path = tmp_path / "vocals.wav"
+    sf.write(path, audio.astype(np.float32), sr)
+    segments = [TranscriptSegment(0.0, 2.0, "one two")]
+    spans = REAL_UNTRANSCRIBED_SINGING(path, segments)
+    assert len(spans) == 1
+    assert spans[0][0] == pytest.approx(4.0, abs=0.1)
+    assert spans[0][1] == pytest.approx(9.0, abs=0.1)
+
+
+def test_lines_whisper_missed_are_aligned_in_the_untranscribed_singing(
+        tmp_path, monkeypatch):
+    lyrics = "one two three\nfour five six\nseven eight nine"
+    transcript = [{"start": 1.0, "end": 3.0, "text": "one two three"},
+                  {"start": 40.0, "end": 42.0, "text": "seven eight nine"}]
+    monkeypatch.setattr(align, "untranscribed_singing",
+                        lambda vocals, segments: [(20.0, 26.0)])
+    rows = run_align(tmp_path, monkeypatch, lyrics, transcript)
+    check_invariants(rows, lyrics)
+    middle = [r for r in rows if r["line_index"] == 1]
+    assert all(19.0 <= r["start"] and r["end"] <= 27.0 for r in middle)
+    meta = json.loads((tmp_path / "w" / "align_meta.json").read_text())
+    assert meta["dropped_lines"] == []
+
+
+def test_stars_bracket_each_line_and_take_no_words(tmp_path, monkeypatch):
+    lyrics = "one two\n...\nthree four"
+    transcript = [{"start": 0.0, "end": 10.0, "text": "one two three four"}]
+    seen = []
+
+    def spy(vocals_path, start, end, tokens, device):
+        seen.append(tokens)
+        # each star claims a long stretch, so a word given a star's span shows
+        spans = [(start + i, start + i + (5.0 if t == "*" else 0.5), 0.9)
+                 for i, t in enumerate(tokens)]
+        return spans
+
+    monkeypatch.setattr(align, "force_align_section", spy)
+    ctx = make_ctx(tmp_path, lyrics, transcript)
+    align.align_stage().run(ctx)
+    rows = json.loads((ctx.work / "alignment.json").read_text())
+    assert seen == [["*", "one", "two", "*", "three", "four", "*"]]
+    timed = [r for r in rows if r["word"] != "..."]
+    assert all(r["end"] - r["start"] == pytest.approx(0.5) for r in timed)
+    check_invariants(rows, lyrics)

tests/test_notes.py

@@ -4,7 +4,7 @@ import numpy as np
 import pytest
 from songtrace.config import Config
 from songtrace.notes import (
-    MIN_NOTE_SEC, TimedNote, build_notes, hz_to_ultrastar, segment_span,
+    MIN_NOTE_SEC, TimedNote, _warnings, build_notes, hz_to_ultrastar, segment_span,
     syllabify,
 )
 from songtrace.pipeline import Ctx, SongInput
@@ -127,11 +127,36 @@ def test_build_notes_unvoiced_word_flagged():
     assert warnings[0] == "1 low-confidence note near 0:01–0:01"
 
 
-def test_build_notes_low_score_word_flagged():
+def _scored_rows(scores, dur=0.4):
+    return [{"word": w, "start": 1.0 + i, "end": 1.0 + i + dur, "score": sc,
+             "line_index": 0}
+            for i, (w, sc) in enumerate(zip(["one", "two", "three", "four", "five"], scores))]
+
+
+def test_build_notes_flags_a_word_scoring_far_below_the_rest():
     times, f0, conf = flat_pitch()
-    rows = [{"word": "hey", "start": 1.0, "end": 1.4, "score": 0.1, "line_index": 0}]
+    notes, _, warnings = build_notes(_scored_rows([0.4, 0.4, 0.05, 0.4, 0.4]),
-    notes, _, warnings = build_notes(rows, times, f0, conf, "English")
+                                     times, f0, conf, "English")
-    assert all(n.low_confidence for n in notes) and len(warnings) == 1
+    assert [n.low_confidence for n in notes] == [False, False, True, False, False]
+    assert len(warnings) == 1
+
+
+def test_build_notes_judges_scores_against_the_song_not_a_fixed_cutoff():
+    # sung vocals score low throughout; uniformly low is not a reason to flag
+    times, f0, conf = flat_pitch()
+    notes, _, warnings = build_notes(_scored_rows([0.05] * 5), times, f0, conf, "English")
+    assert not any(n.low_confidence for n in notes) and warnings == []
+
+
+def test_build_notes_flags_words_too_short_or_too_long_to_be_sung():
+    times, f0, conf = flat_pitch()
+    for dur in (0.05, 3.5):
+        rows = _scored_rows([0.4] * 3)
+        rows[1]["end"] = rows[1]["start"] + dur
+        rows[2]["start"], rows[2]["end"] = rows[1]["end"] + 0.1, rows[1]["end"] + 0.5
+        notes, _, _ = build_notes(rows, times, f0, conf, "English")
+        # flat pitch: one note per word
+        assert [n.low_confidence for n in notes] == [False, True, False], dur
 
 
 def test_build_notes_unvoiced_word_inherits_previous_pitch():
@@ -334,3 +359,13 @@ def test_notes_carry_line_index():
     rows = [{"word": "hey", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 3}]
     notes, _, _ = build_notes(rows, times, f0, conf, "English")
     assert notes[0].line_index == 3
+
+
+def test_warnings_group_flagged_notes_close_together():
+    def note(start, low):
+        return TimedNote(start, 0.3, 0, "la", low, 0)
+    notes = [note(10.0, True), note(10.5, False), note(11.0, True),   # 0.7s apart
+             note(20.0, True)]                                       # far from both
+    assert _warnings(notes) == [
+        "2 low-confidence notes near 0:10–0:11",
+        "1 low-confidence note near 0:20–0:20"]

tests/test_transcribe.py

@@ -1,6 +1,7 @@
 from songtrace.lyrics import normalize_lyrics
 from songtrace.whisper_client import TranscriptSegment
-from songtrace.stages.transcribe import classify_lyrics_match, lyrics_similarity, vocal_sections
+from songtrace.stages.transcribe import (classify_lyrics_match, lyrics_similarity,
+                                         recovered_spans, vocal_sections)
 
 def seg(s, e, t=""):
     return TranscriptSegment(start=s, end=e, text=t)
@@ -64,3 +65,31 @@ def test_classify_lyrics_match_scopes_missing_span_to_adlib_not_whole_section():
     assert missing == [(5.0, 9.0)]
     for a, b in missing:
         assert not (a <= 1.0 < b)        # matched line's start (1.0) untouched
+
+
+THREE_LINES = "one two three\nfour five six\nseven eight nine"
+
+
+def test_lines_whisper_missed_are_kept_when_singing_went_untranscribed():
+    ref = normalize_lyrics(THREE_LINES)
+    segs = [seg(0, 2, "one two three"), seg(20, 22, "seven eight nine")]
+    assert classify_lyrics_match(segs, ref)[0] == [1]
+    dropped, _ = classify_lyrics_match(segs, ref, unheard=[(8.0, 14.0)])
+    assert dropped == []
+
+
+def test_untranscribed_singing_elsewhere_does_not_keep_a_line():
+    ref = normalize_lyrics(THREE_LINES)
+    segs = [seg(0, 2, "one two three"), seg(20, 22, "seven eight nine")]
+    dropped, _ = classify_lyrics_match(segs, ref, unheard=[(30.0, 35.0)])
+    assert dropped == [1]
+
+
+def test_recovered_spans_are_the_singing_between_heard_neighbours():
+    ref = normalize_lyrics(THREE_LINES)
+    segs = [seg(10, 12, "four five six"), seg(20, 22, "seven eight nine")]
+    unheard = [(3.0, 9.0), (14.0, 18.0), (30.0, 35.0)]
+    # line 0 comes before anything heard, so only the singing before line 1
+    # can stand in for it; the other spans have no missing line around them
+    assert recovered_spans(segs, ref, unheard) == [(3.0, 9.0)]
+    assert classify_lyrics_match(segs, ref, unheard=unheard)[0] == []

tests/test_web.py

@@ -834,7 +834,7 @@ def test_narrow_stage_labels_stay_in_the_accessibility_tree():
 
 def test_rebuild_mode_reuses_the_one_form():
     html = _index_html()
-    assert html.count("<form") == 1
+    assert html.count('name="lyrics"') == 1    # one song form, not a rebuild copy
     assert _rebuild_flaws(html) == []
 
 
@@ -1746,3 +1746,109 @@ def test_submitting_crlf_lyrics_stores_them_with_lf(tmp_path):
     client, _ = make_client(tmp_path, ok_runner)
     job_id = make_done_job(client, lyrics="one\r\ntwo")
     assert client.get(f"/api/jobs/{job_id}/input").json()["lyrics"] == "one\ntwo"
+
+
+def test_rebuild_all_rebuilds_every_song_from_the_stage(tmp_path):
+    seen = []
+    client, _ = make_client(tmp_path, capture_runner_factory(seen))
+    first = make_done_job(client, title="One")
+    second = make_done_job(client, title="Two")
+    resp = client.post("/api/rebuild-all", data={"force_from": "pitch"})
+    assert resp.status_code == 201
+    body = resp.json()
+    assert len(body["ids"]) == 2 and body["skipped"] == []
+    jobs = [wait_status(client, i, "done") for i in body["ids"]]
+    assert {j["rebuilt_from"] for j in jobs} == {first, second}
+    assert [s["force_from"] for s in seen[2:]] == ["pitch", "pitch"]
+    by_title = lambda runs: sorted((r["inp"] for r in runs), key=lambda i: i.title)
+    assert by_title(seen[2:]) == by_title(seen[:2])
+
+
+def test_rebuild_all_skips_jobs_that_were_already_rebuilt(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    parent = make_done_job(client)
+    renamed = client.post(f"/api/jobs/{parent}/rebuild",
+                          data=rebuild_form(title="Renamed")).json()["id"]
+    wait_status(client, renamed, "done")
+    body = client.post("/api/rebuild-all", data={"force_from": "notes"}).json()
+    assert len(body["ids"]) == 1
+    job = wait_status(client, body["ids"][0], "done")
+    assert job["rebuilt_from"] == renamed
+
+
+def test_rebuild_all_takes_the_latest_job_for_a_song(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    make_done_job(client)
+    latest = make_done_job(client)
+    body = client.post("/api/rebuild-all", data={"force_from": "grid"}).json()
+    assert len(body["ids"]) == 1
+    assert wait_status(client, body["ids"][0], "done")["rebuilt_from"] == latest
+
+
+def test_rebuild_all_skips_songs_in_progress(tmp_path):
+    gate = threading.Event()
+
+    def blocking_runner(inp, cfg, log=print, force_from=None, stages=None):
+        gate.wait(timeout=5)
+        return PipelineResult(output_dir=cfg.output_dir / song_slug(inp))
+
+    client, _ = make_client(tmp_path, blocking_runner)
+    try:
+        busy = client.post("/api/jobs", data={
+            "title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"}).json()["id"]
+        body = client.post("/api/rebuild-all", data={"force_from": "pitch"}).json()
+        assert body == {"ids": [], "skipped": [{"id": busy, "reason": "still in progress"}]}
+    finally:
+        gate.set()
+
+
+def test_rebuild_all_from_acquire_skips_a_song_whose_upload_is_gone(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    job_id = client.post("/api/jobs", data={"title": "T", "artist": "A", "lyrics": "x"},
+                         files={"video": ("v.mp4", io.BytesIO(b"data"), "video/mp4")}
+                         ).json()["id"]
+    wait_status(client, job_id, "done")
+    shutil.rmtree(tmp_path / "work" / "uploads", ignore_errors=True)
+    body = client.post("/api/rebuild-all", data={"force_from": "acquire"}).json()
+    assert body["ids"] == []
+    assert body["skipped"][0]["id"] == job_id
+    assert "upload" in body["skipped"][0]["reason"]
+
+
+def test_rebuild_all_rejects_an_unknown_stage(tmp_path):
+    client, _ = make_client(tmp_path, ok_runner)
+    make_done_job(client)
+    assert client.post("/api/rebuild-all", data={"force_from": "nope"}).status_code == 400
+    assert client.post("/api/rebuild-all", data={}).status_code == 422
+
+
+def test_rebuild_all_lists_the_current_job_of_each_song(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    one = make_done_job(client, title="One")
+    two = make_done_job(client, title="Two")
+    rerun = client.post(f"/api/jobs/{one}/rerun").json()["id"]
+    wait_status(client, rerun, "done")
+    songs = client.get("/api/rebuild-all").json()
+    assert [(s["id"], s["title"], s["rebuilt_from"]) for s in songs] == [
+        (rerun, "One", one), (two, "Two", None)]
+    assert all(s["status"] == "done" and s["finished_at"] for s in songs)
+
+
+def test_rebuild_all_rebuilds_only_the_songs_selected(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    make_done_job(client, title="One")
+    two = make_done_job(client, title="Two")
+    resp = client.post("/api/rebuild-all", data={"force_from": "pitch", "ids": [two]})
+    body = resp.json()
+    assert len(body["ids"]) == 1 and body["skipped"] == []
+    assert wait_status(client, body["ids"][0], "done")["rebuilt_from"] == two
+
+
+def test_rebuild_all_reports_a_selected_job_that_was_since_rebuilt(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    old = make_done_job(client)
+    rerun = client.post(f"/api/jobs/{old}/rerun").json()["id"]
+    wait_status(client, rerun, "done")
+    body = client.post("/api/rebuild-all", data={"force_from": "pitch", "ids": [old]}).json()
+    assert body == {"ids": [], "skipped": [
+        {"id": old, "reason": "no longer the current job for its song"}]}