ci: upgrade pip in the venv before installing torch

Debian bookworm's python3-venv seeds pip 23.0.1, which rejects the
PyTorch CPU index's typing_extensions wheel because its metadata spells
the name with an underscore. It then falls back to the sdist, whose build
needs flit_core, which that index does not carry.

.dejima/allowed-domains

@@ -0,0 +1,6 @@
+deb.debian.org
+security.debian.org
+pypi.org
+files.pythonhosted.org
+download.pytorch.org
+download-r2.pytorch.org

.gitea/workflows/tests.yml

@@ -0,0 +1,86 @@
+name: tests
+
+# `pull_request_target` reads this file from the base branch, so a pull
+# request cannot change what runs or where (including `runs-on:`). The run's
+# ref is therefore the base branch, and the checkout has to name the head.
+on:
+  pull_request_target:
+
+jobs:
+  test:
+    # No Docker socket, and egress limited to .dejima/allowed-domains on main.
+    runs-on: dejima-confined
+
+    # Node for actions/checkout; Debian bookworm's python3 is the 3.11 the
+    # Dockerfile ships.
+    container:
+      image: node:22-bookworm
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      # Inline rather than a script in the repository: the files come from the
+      # branch under test, so a script would be missing from older branches.
+      - name: Check the hosts this job fetches from are reachable
+        run: |
+          set -u
+          bad=""
+          for host in deb.debian.org security.debian.org pypi.org \
+                      files.pythonhosted.org download.pytorch.org \
+                      download-r2.pytorch.org; do
+            resolved=""
+            for attempt in 1 2 3 4; do
+              getent hosts "$host" >/dev/null 2>&1 && { resolved=yes; break; }
+              sleep 2
+            done
+            if [ -z "$resolved" ]; then
+              echo "no    $host - does not resolve"; bad="$bad $host(dns)"; continue
+            fi
+            # Any HTTP answer proves the connection; curl reports 000 for none.
+            code="$(curl -sS -o /dev/null --max-time 20 --retry 2 -w '%{http_code}' "https://$host/" 2>/dev/null || true)"
+            case "${code:-000}" in
+              000) echo "no    $host - resolves, but nothing answered over https"
+                   bad="$bad $host(net)" ;;
+              *)   echo "ok    $host (http $code)" ;;
+            esac
+          done
+          [ -z "$bad" ] && exit 0
+          {
+            echo
+            echo "Unreachable:$bad"
+            echo
+            echo "The confined tier reaches only what .dejima/allowed-domains on"
+            echo "main declares, rendered onto the runner separately from the"
+            echo "merge. On the runner:"
+            echo "  sudo dejima-ci-allowlist show    - is the name listed?"
+            echo "  sudo dejima-ci-allowlist check   - is a change pending?"
+            echo "  sudo dejima-ci-allowlist render  - apply it now"
+            echo "A (dns) failure is the resolver; a (net) one is the packet"
+            echo "filter, which sudo dejima-ci-firewall reapplies."
+            echo
+            echo "download.pytorch.org redirects its wheels to"
+            echo "download-r2.pytorch.org, so both have to be allowed."
+          } >&2
+          exit 1
+
+      - name: Install Python
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends python3 python3-venv
+          python3 -m venv /opt/venv
+          # bookworm's pip 23.0 rejects the PyTorch index's typing_extensions
+          # wheel over its name's spelling, then cannot build it from source
+          /opt/venv/bin/pip install --retries 5 --upgrade pip
+          echo /opt/venv/bin >> "$GITHUB_PATH"
+
+      # The CPU build first: torch from PyPI is the CUDA build, several GB of
+      # libraries no test uses. The install below then finds torch satisfied.
+      - name: Install CPU-only torch
+        run: pip install --retries 5 torch torchaudio --index-url https://download.pytorch.org/whl/cpu
+
+      - name: Install songtrace
+        run: pip install --retries 5 -e '.[dev]'
+
+      - run: pytest

README.md

@@ -177,6 +177,12 @@ to redo a stage anyway — pick `package` to repackage without re-downloading.
 It can only widen what is rerun, so an edit that invalidates an earlier stage
 still wins.
 
+**REBUILD ALL FROM…**, under CREATE SONG, does the same for every song at once:
+pick the stage an update improved and each song is rebuilt from there with the
+inputs it has now. Every song starts ticked; untick any already rebuilt by hand
+(each shows when it was last built). Songs still in progress are skipped, as
+are songs whose uploaded video is gone when the stage is `acquire`.
+
 ## YouTube download failures
 
 `HTTP Error 403: Forbidden` from the acquire stage means YouTube served the

src/songtrace/stages/pitch.py

@@ -6,6 +6,11 @@ import numpy as np
 from ..models import on_release
 from ..pipeline import Ctx, Stage
 
+# Separated vocal stems keep faint bleed that crepe still finds a pitch in, so
+# frames this far below the song's loud vocal level count as silence. Relative
+# rather than absolute, so it holds however loud the song was mastered.
+SILENCE_BELOW_LOUD_DB = 25.0
+
 
 @on_release
 def unload_model() -> None:
@@ -18,6 +23,11 @@ def unload_model() -> None:
             delattr(torchcrepe.infer, attr)
 
 
+def mute_silence(conf: np.ndarray, loudness: np.ndarray) -> None:
+    """Zero the confidence of frames quiet enough to be bleed, in place."""
+    conf[loudness < np.quantile(loudness, 0.9) - SILENCE_BELOW_LOUD_DB] = 0.0
+
+
 def track_pitch(vocals_path: Path, model: str, device: str):
     import torchaudio
     import torchcrepe
@@ -33,6 +43,7 @@ def track_pitch(vocals_path: Path, model: str, device: str):
         return_periodicity=True, batch_size=512)
     f0 = f0[0].cpu().numpy()
     conf = periodicity[0].cpu().numpy()
+    mute_silence(conf, torchcrepe.loudness.a_weighted(waveform, 16000, 160)[0].cpu().numpy())
     times = np.arange(len(f0)) * 0.01
     return times, f0, conf
 

src/songtrace/static/index.html

@@ -113,6 +113,61 @@ body {
 #sidebar .eyebrow, #jobs .empty-note { padding: 0 18px; }
 #create-song { display: block; width: calc(100% - 36px); margin: 0 18px 20px; padding: 9px 12px; }
 
+#rebuild-all { display: block; margin: -8px 18px 20px; }
+#rebuild-all-note { margin: -12px 18px 20px; }
+
+#rebuild-all-dialog {
+  width: min(480px, calc(100vw - 32px));
+  padding: 26px 28px;
+  color: var(--text);
+  background: var(--panel);
+  border: 1px solid var(--border);
+}
+#rebuild-all-dialog::backdrop { background: rgb(10 10 15 / 0.75); }
+#rebuild-all-dialog h2 {
+  margin: 0 0 14px;
+  font-family: var(--font-display);
+  font-weight: 700;
+  font-size: 0.8rem;
+  letter-spacing: 0.22em;
+  color: var(--text-muted);
+}
+#rebuild-all-form { display: grid; gap: 16px; }
+#rebuild-all-form p { margin: 0; font-size: 0.88rem; line-height: 1.5; }
+#rebuild-all-count { color: var(--warn); }
+#rebuild-all-songs { margin: 0; padding: 0; border: 0; min-width: 0; }
+#rebuild-all-songs legend {
+  margin-bottom: 6px;
+  padding: 0;
+  font-family: var(--font-display);
+  font-weight: 600;
+  font-size: 0.68rem;
+  letter-spacing: 0.14em;
+  text-transform: uppercase;
+  color: var(--text-muted);
+}
+#rebuild-all-list {
+  max-height: 40vh;
+  overflow-y: auto;
+  border-top: 1px solid var(--border);
+  border-bottom: 1px solid var(--border);
+}
+.rebuild-all-row {
+  display: grid;
+  grid-template-columns: auto 1fr;
+  gap: 1px 10px;
+  align-items: baseline;
+  padding: 7px 4px;
+  font-size: 0.88rem;
+  cursor: pointer;
+}
+.rebuild-all-row input { accent-color: var(--purple); }
+.rebuild-all-row:has(input:disabled) { cursor: default; color: var(--text-muted); }
+.rebuild-all-row .meta { grid-column: 2; font-size: 0.76rem; color: var(--text-muted); }
+.rebuild-all-every { padding-top: 0; color: var(--text-muted); }
+#rebuild-all-error { min-height: 1.2em; color: var(--fail); }
+#rebuild-all-error:empty { display: none; }
+
 #jobs { display: flex; flex-direction: column; gap: 1px; margin-top: 8px; }
 
 .job {
@@ -518,11 +573,39 @@ button { cursor: pointer; }
   </div>
   <div>
     <button type="button" id="create-song" class="primary">CREATE SONG</button>
+    <button type="button" id="rebuild-all" class="action">REBUILD ALL FROM…</button>
+    <p id="rebuild-all-note" class="empty-note" role="status" hidden></p>
     <h2 class="eyebrow">QUEUE</h2>
     <div id="jobs"></div>
   </div>
 </aside>
 
+<dialog id="rebuild-all-dialog" aria-labelledby="rebuild-all-heading">
+  <form id="rebuild-all-form" method="dialog">
+    <h2 id="rebuild-all-heading">REBUILD ALL SONGS</h2>
+    <p>Reruns the songs ticked below from the chosen stage onwards with the
+      inputs they have now, for when a songtrace update improves that stage.</p>
+    <label class="field">
+      <span>Restart from</span>
+      <select name="force_from" required></select>
+    </label>
+    <fieldset id="rebuild-all-songs">
+      <legend>Songs</legend>
+      <label class="rebuild-all-row rebuild-all-every">
+        <input type="checkbox" id="rebuild-all-toggle" checked>
+        <span>All songs</span>
+      </label>
+      <div id="rebuild-all-list"></div>
+    </fieldset>
+    <p id="rebuild-all-count"></p>
+    <p id="rebuild-all-error" role="alert"></p>
+    <div class="form-actions">
+      <button type="submit" class="primary">REBUILD SELECTED</button>
+      <button type="button" class="action" id="rebuild-all-cancel">CANCEL</button>
+    </div>
+  </form>
+</dialog>
+
 <main id="main">
   <section id="new-song">
     <h2>NEW SONG</h2>
@@ -702,6 +785,95 @@ createSong.addEventListener("click", () => {
   form.elements.title.focus();
 });
 
+const rebuildAllButton = document.getElementById("rebuild-all");
+const rebuildAllNote = document.getElementById("rebuild-all-note");
+const rebuildAllDialog = document.getElementById("rebuild-all-dialog");
+const rebuildAllForm = document.getElementById("rebuild-all-form");
+const rebuildAllCount = document.getElementById("rebuild-all-count");
+const rebuildAllError = document.getElementById("rebuild-all-error");
+
+const rebuildAllList = document.getElementById("rebuild-all-list");
+const rebuildAllToggle = document.getElementById("rebuild-all-toggle");
+const rebuildAllSubmit = rebuildAllForm.querySelector("button[type=submit]");
+
+function rebuildAllBoxes() {
+  return [...rebuildAllList.querySelectorAll("input:not(:disabled)")];
+}
+
+function updateRebuildAllCount() {
+  const boxes = rebuildAllBoxes();
+  const chosen = boxes.filter(b => b.checked).length;
+  rebuildAllToggle.checked = chosen === boxes.length && chosen > 0;
+  rebuildAllToggle.indeterminate = chosen > 0 && chosen < boxes.length;
+  rebuildAllCount.textContent = chosen === 1 ? "1 song will be rebuilt."
+                                             : `${chosen} songs will be rebuilt.`;
+  // with nothing ticked no ids are sent, which the API reads as every song
+  rebuildAllSubmit.disabled = chosen === 0;
+}
+
+function rebuildAllRow(song) {
+  const busy = song.status === "queued" || song.status === "running";
+  const row = document.createElement("label");
+  row.className = "rebuild-all-row";
+  const box = document.createElement("input");
+  box.type = "checkbox";
+  box.name = "ids";
+  box.value = song.id;
+  box.checked = !busy;
+  box.disabled = busy;
+  const name = document.createElement("span");
+  name.textContent = `${song.title} · ${song.artist}`;
+  const meta = document.createElement("span");
+  meta.className = "meta";
+  const when = song.finished_at && new Date(song.finished_at * 1000).toLocaleString(
+    undefined, { dateStyle: "medium", timeStyle: "short" });
+  meta.textContent = busy ? "in progress, skipped"
+    : `${song.status === "failed" ? "failed" : song.rebuilt_from ? "rebuilt" : "built"} ${when || ""}`.trim();
+  row.append(box, name, meta);
+  return row;
+}
+
+rebuildAllButton.addEventListener("click", async () => {
+  rebuildAllError.textContent = "";
+  rebuildAllForm.reset();
+  const resp = await fetch("/api/rebuild-all");
+  const songs = resp.ok ? await resp.json() : [];
+  rebuildAllList.replaceChildren(...songs.map(rebuildAllRow));
+  updateRebuildAllCount();
+  rebuildAllDialog.showModal();
+});
+
+rebuildAllList.addEventListener("change", updateRebuildAllCount);
+rebuildAllToggle.addEventListener("change", () => {
+  for (const box of rebuildAllBoxes()) box.checked = rebuildAllToggle.checked;
+  updateRebuildAllCount();
+});
+
+document.getElementById("rebuild-all-cancel").addEventListener(
+  "click", () => rebuildAllDialog.close());
+
+rebuildAllForm.addEventListener("submit", async (e) => {
+  e.preventDefault();
+  const stage = rebuildAllForm.elements.force_from.value;
+  const resp = await fetch("/api/rebuild-all", {
+    method: "POST", body: new FormData(rebuildAllForm) });
+  if (!resp.ok) {
+    const body = await resp.json().catch(() => ({}));
+    rebuildAllError.textContent = body.detail || `error: ${resp.status}`;
+    return;
+  }
+  const { ids, skipped } = await resp.json();
+  rebuildAllDialog.close();
+  let note = `Queued ${ids.length} rebuild${ids.length === 1 ? "" : "s"} from ${stage}.`;
+  if (skipped.length) {
+    const reasons = [...new Set(skipped.map(s => s.reason))].join("; ");
+    note += ` Skipped ${skipped.length}: ${reasons}.`;
+  }
+  rebuildAllNote.textContent = note;
+  rebuildAllNote.hidden = false;
+  refreshJobs();
+});
+
 function jobCard(job) {
   const card = document.createElement("button");
   card.type = "button";
@@ -780,6 +952,8 @@ async function loadStages() {
   const select = form.elements.force_from;
   select.replaceChildren(new Option("only what changed", ""),
                          ...STAGES.map(s => new Option(s, s)));
+  rebuildAllForm.elements.force_from.replaceChildren(
+    new Option("choose a stage", ""), ...STAGES.map(s => new Option(s, s)));
 }
 
 function renderStageTrack(job) {

src/songtrace/web.py

@@ -329,6 +329,51 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
             rebuild_policy.plan(parent.inp, inp), force_from))
         return _enqueue(new_id, inp, rebuilt_from=job_id, force_stage=force_from)
 
+    def _current_songs() -> list[Job]:
+        """The job that stands for each song now, newest first."""
+        with lock:
+            # a job that was rebuilt is history: its rebuild is the song now,
+            # possibly under another slug
+            rebuilt = {j.rebuilt_from for j in jobs.values()}
+            latest = {song_slug(j.inp): j for j in (jobs[i] for i in order)
+                      if j.id not in rebuilt}
+            position = {job_id: n for n, job_id in enumerate(order)}
+            return sorted(latest.values(), key=lambda j: position[j.id], reverse=True)
+
+    @app.get("/api/rebuild-all")
+    def rebuild_all_candidates():
+        return [{"id": j.id, "title": j.title, "artist": j.artist,
+                 "status": j.status, "rebuilt_from": j.rebuilt_from,
+                 "finished_at": j.finished_at}
+                for j in _current_songs()]
+
+    @app.post("/api/rebuild-all", status_code=201)
+    def rebuild_all(force_from: str = Form(...), ids: list[str] | None = Form(None)):
+        """Rebuild every song, or the ones in `ids`, from `force_from` on, for
+        when songtrace itself improved that stage. Each is an ordinary rebuild
+        with unchanged inputs."""
+        if force_from not in STAGE_ORDER:
+            raise HTTPException(400, f"unknown stage {force_from!r}")
+        songs = _current_songs()
+        queued, skipped = [], []
+        if ids is not None:
+            current = {j.id for j in songs}
+            skipped = [{"id": i, "reason": "no longer the current job for its song"}
+                       for i in ids if i not in current]
+            songs = [j for j in songs if j.id in set(ids)]
+        for job in songs:
+            if job.status in ("queued", "running"):
+                skipped.append({"id": job.id, "reason": "still in progress"})
+                continue
+            try:
+                _require_source(job.inp, job.inp, force_from)
+            except HTTPException as e:
+                skipped.append({"id": job.id, "reason": e.detail})
+                continue
+            queued.append(_enqueue(uuid.uuid4().hex, job.inp, rebuilt_from=job.id,
+                                   force_stage=force_from)["id"])
+        return {"ids": queued, "skipped": skipped}
+
     @app.get("/api/stages")
     def list_stages():
         return {"stages": list(STAGE_ORDER)}

tests/fixtures/chart_crlf.txt

@@ -0,0 +1,36 @@
+#TITLE:Placeholder Chart
+#ARTIST:Test Artist
+#LANGUAGE:English
+#YEAR:2000
+#CREATOR:songtrace tests
+#GENRE:Test
+#COVER:Test Artist - Placeholder Chart [CO].jpg
+#MP3:Test Artist - Placeholder Chart.mp3
+#VIDEO:Test Artist - Placeholder Chart.mp4
+#BPM:217,98
+#GAP:4880
+#MEDLEYSTARTBEAT:113
+#MEDLEYENDBEAT:240
+: 0 3 16 ah
+: 5 3 20  ah
+: 11 3 23  la
+: 17 3 16  la
+: 22 28 15  ah
+: 51 2 16  doo
+: 54 31 16 ~
+- 95  
+F 113 3 18 la
+F 117 3 21  la
+F 121 3 21  la
+* 125 3 21  doo
+* 129 12 19 ~
+- 150
+: 155 4 12 Na
+R 160 4 0  na
+G 165 4 0  na
+: 170 6 14  la
+- 190
+: 194 3 16 La
+: 198 3 18  la
+* 202 10 20  ah
+E

tests/fixtures/chart_latin1.txt

@@ -0,0 +1,21 @@
+#TITLE:Café Placeholder
+#ARTIST:Tést Artist
+#LANGUAGE:English
+#GENRE:Test
+#YEAR:2000
+#MP3:Tést Artist - Café Placeholder.mp3
+#CREATOR:songtrace tests
+#BPM:256
+#GAP:49673
+: 0 4 2 Doo
+: 6 4 4  doo
+: 12 8 5  laé
+- 28 30
+: 32 4 0 Na
+: 38 4 -2  na
+* 44 12 -9  ah
+- 64
+F 70 4 3 La
+: 76 6 -5  la
+: 84 10 -9 ~
+E

tests/test_notes.py

@@ -78,7 +78,7 @@ def test_syllabify():
     assert syllabify("reminds", "English") == ["re", "minds"]
     assert syllabify("a", "English") == ["a"]
     assert syllabify("xyzzy", None) == ["xyzzy"]
-    assert syllabify("She's", "English") == ["She's"]
+    assert syllabify("It's", "English") == ["It's"]
 
 
 def test_syllabify_preserves_word_characters():
@@ -96,12 +96,12 @@ def test_syllabify_language_without_pyphen_dict_is_not_mishandled_as_english():
 def test_build_notes_basic_line():
     times, f0, conf = flat_pitch()
     rows = [
-        {"word": "She's", "start": 1.0, "end": 1.5, "score": 0.9, "line_index": 0},
+        {"word": "It's", "start": 1.0, "end": 1.5, "score": 0.9, "line_index": 0},
         {"word": "got", "start": 1.6, "end": 2.0, "score": 0.9, "line_index": 0},
         {"word": "Now", "start": 5.0, "end": 5.5, "score": 0.9, "line_index": 1},
     ]
     notes, breaks, warnings = build_notes(rows, times, f0, conf, "English")
-    assert notes[0].text == "She's"          # line-initial: no leading space
+    assert notes[0].text == "It's"          # line-initial: no leading space
     assert notes[1].text == " got"           # mid-line: leading space
     assert notes[2].text == "Now"            # new line: no leading space
     assert len(breaks) == 1 and 2.0 <= breaks[0] <= 5.0
@@ -278,7 +278,7 @@ def test_notes_stage_writes_json(tmp_path):
     times, f0, conf = flat_pitch()
     np.savez(work / "pitch.npz", times=times, f0=f0, conf=conf)
     (work / "alignment.json").write_text(json.dumps([
-        {"word": "She's", "start": 1.0, "end": 1.5, "score": 0.9, "line_index": 0},
+        {"word": "It's", "start": 1.0, "end": 1.5, "score": 0.9, "line_index": 0},
         {"word": "gone", "start": 3.0, "end": 3.5, "score": 0.9, "line_index": 1},
     ]))
     ctx = Ctx(inp=SongInput(title="T", artist="A", lyrics_text="x", language="English"),
@@ -287,7 +287,7 @@ def test_notes_stage_writes_json(tmp_path):
     assert stage.outputs(ctx) == [work / "notes_sec.json"]
     stage.run(ctx)
     data = json.loads((work / "notes_sec.json").read_text())
-    assert [n["text"] for n in data["notes"]] == ["She's", "gone"]
+    assert [n["text"] for n in data["notes"]] == ["It's", "gone"]
     assert data["breaks"] == [2.25]
     assert data["warnings"] == []
     assert TimedNote(**data["notes"][0]).pitch == 0

tests/test_pitch.py

@@ -17,3 +17,21 @@ def test_pitch_stage_writes_npz(tmp_path, monkeypatch):
     data = np.load(work / "pitch.npz")
     assert np.allclose(data["f0"], 220.0)
     assert data["times"].shape == data["conf"].shape
+
+
+def test_mute_silence_drops_bleed_but_keeps_quiet_singing():
+    # 60 frames of singing around -30 dB, one quiet sung frame at -50 dB, and
+    # 40 frames of bleed at -60 dB: only the bleed is more than 25 dB below
+    # the song's loud level.
+    loudness = np.array([-30.0] * 60 + [-50.0] + [-60.0] * 40)
+    conf = np.full_like(loudness, 0.8)
+    pitch.mute_silence(conf, loudness)
+    assert (conf[:61] == 0.8).all()
+    assert (conf[61:] == 0.0).all()
+
+
+def test_mute_silence_is_relative_to_the_songs_level():
+    loudness = np.array([-45.0] * 60 + [-75.0] * 40)
+    conf = np.full_like(loudness, 0.8)
+    pitch.mute_silence(conf, loudness)
+    assert (conf[:60] == 0.8).all() and (conf[60:] == 0.0).all()

tests/test_ultrastar.py

@@ -10,8 +10,8 @@ def load(name):
     return (FIXTURES / name).read_text(encoding="utf-8", errors="replace")
 
 def test_parse_headers_and_notes():
-    song = parse_ultrastar(load("sanctuary.txt"))
-    assert get_header(song, "artist") == "Utada Hikaru"
+    song = parse_ultrastar(load("chart_crlf.txt"))
+    assert get_header(song, "artist") == "Test Artist"
     assert parse_bpm(get_header(song, "BPM")) == 217.98
     first = song.body[0]
     assert first == Note(kind=":", beat=0, length=3, pitch=16, text="ah")
@@ -19,16 +19,16 @@ def test_parse_headers_and_notes():
     assert any(n.kind == "F" for n in song.body if isinstance(n, Note))
 
 def test_note_text_keeps_leading_space():
-    song = parse_ultrastar(load("sanctuary.txt"))
+    song = parse_ultrastar(load("chart_crlf.txt"))
     notes = [n for n in song.body if isinstance(n, Note)]
     assert notes[1].text == " ah"
 
 def test_negative_pitch():
-    song = parse_ultrastar(load("sweet_child.txt"))
+    song = parse_ultrastar(load("chart_latin1.txt"))
     assert any(n.pitch == -9 for n in song.body if isinstance(n, Note))
 
 def test_round_trip():
-    for name in ("sweet_child.txt", "sanctuary.txt"):
+    for name in ("chart_latin1.txt", "chart_crlf.txt"):
         song = parse_ultrastar(load(name))
         again = parse_ultrastar(serialize_ultrastar(song))
         assert again == song

tests/test_web.py

@@ -834,7 +834,7 @@ def test_narrow_stage_labels_stay_in_the_accessibility_tree():
 
 def test_rebuild_mode_reuses_the_one_form():
     html = _index_html()
-    assert html.count("<form") == 1
+    assert html.count('name="lyrics"') == 1    # one song form, not a rebuild copy
     assert _rebuild_flaws(html) == []
 
 
@@ -1746,3 +1746,109 @@ def test_submitting_crlf_lyrics_stores_them_with_lf(tmp_path):
     client, _ = make_client(tmp_path, ok_runner)
     job_id = make_done_job(client, lyrics="one\r\ntwo")
     assert client.get(f"/api/jobs/{job_id}/input").json()["lyrics"] == "one\ntwo"
+
+
+def test_rebuild_all_rebuilds_every_song_from_the_stage(tmp_path):
+    seen = []
+    client, _ = make_client(tmp_path, capture_runner_factory(seen))
+    first = make_done_job(client, title="One")
+    second = make_done_job(client, title="Two")
+    resp = client.post("/api/rebuild-all", data={"force_from": "pitch"})
+    assert resp.status_code == 201
+    body = resp.json()
+    assert len(body["ids"]) == 2 and body["skipped"] == []
+    jobs = [wait_status(client, i, "done") for i in body["ids"]]
+    assert {j["rebuilt_from"] for j in jobs} == {first, second}
+    assert [s["force_from"] for s in seen[2:]] == ["pitch", "pitch"]
+    by_title = lambda runs: sorted((r["inp"] for r in runs), key=lambda i: i.title)
+    assert by_title(seen[2:]) == by_title(seen[:2])
+
+
+def test_rebuild_all_skips_jobs_that_were_already_rebuilt(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    parent = make_done_job(client)
+    renamed = client.post(f"/api/jobs/{parent}/rebuild",
+                          data=rebuild_form(title="Renamed")).json()["id"]
+    wait_status(client, renamed, "done")
+    body = client.post("/api/rebuild-all", data={"force_from": "notes"}).json()
+    assert len(body["ids"]) == 1
+    job = wait_status(client, body["ids"][0], "done")
+    assert job["rebuilt_from"] == renamed
+
+
+def test_rebuild_all_takes_the_latest_job_for_a_song(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    make_done_job(client)
+    latest = make_done_job(client)
+    body = client.post("/api/rebuild-all", data={"force_from": "grid"}).json()
+    assert len(body["ids"]) == 1
+    assert wait_status(client, body["ids"][0], "done")["rebuilt_from"] == latest
+
+
+def test_rebuild_all_skips_songs_in_progress(tmp_path):
+    gate = threading.Event()
+
+    def blocking_runner(inp, cfg, log=print, force_from=None, stages=None):
+        gate.wait(timeout=5)
+        return PipelineResult(output_dir=cfg.output_dir / song_slug(inp))
+
+    client, _ = make_client(tmp_path, blocking_runner)
+    try:
+        busy = client.post("/api/jobs", data={
+            "title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"}).json()["id"]
+        body = client.post("/api/rebuild-all", data={"force_from": "pitch"}).json()
+        assert body == {"ids": [], "skipped": [{"id": busy, "reason": "still in progress"}]}
+    finally:
+        gate.set()
+
+
+def test_rebuild_all_from_acquire_skips_a_song_whose_upload_is_gone(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    job_id = client.post("/api/jobs", data={"title": "T", "artist": "A", "lyrics": "x"},
+                         files={"video": ("v.mp4", io.BytesIO(b"data"), "video/mp4")}
+                         ).json()["id"]
+    wait_status(client, job_id, "done")
+    shutil.rmtree(tmp_path / "work" / "uploads", ignore_errors=True)
+    body = client.post("/api/rebuild-all", data={"force_from": "acquire"}).json()
+    assert body["ids"] == []
+    assert body["skipped"][0]["id"] == job_id
+    assert "upload" in body["skipped"][0]["reason"]
+
+
+def test_rebuild_all_rejects_an_unknown_stage(tmp_path):
+    client, _ = make_client(tmp_path, ok_runner)
+    make_done_job(client)
+    assert client.post("/api/rebuild-all", data={"force_from": "nope"}).status_code == 400
+    assert client.post("/api/rebuild-all", data={}).status_code == 422
+
+
+def test_rebuild_all_lists_the_current_job_of_each_song(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    one = make_done_job(client, title="One")
+    two = make_done_job(client, title="Two")
+    rerun = client.post(f"/api/jobs/{one}/rerun").json()["id"]
+    wait_status(client, rerun, "done")
+    songs = client.get("/api/rebuild-all").json()
+    assert [(s["id"], s["title"], s["rebuilt_from"]) for s in songs] == [
+        (rerun, "One", one), (two, "Two", None)]
+    assert all(s["status"] == "done" and s["finished_at"] for s in songs)
+
+
+def test_rebuild_all_rebuilds_only_the_songs_selected(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    make_done_job(client, title="One")
+    two = make_done_job(client, title="Two")
+    resp = client.post("/api/rebuild-all", data={"force_from": "pitch", "ids": [two]})
+    body = resp.json()
+    assert len(body["ids"]) == 1 and body["skipped"] == []
+    assert wait_status(client, body["ids"][0], "done")["rebuilt_from"] == two
+
+
+def test_rebuild_all_reports_a_selected_job_that_was_since_rebuilt(tmp_path):
+    client, _ = make_client(tmp_path, capture_runner_factory([]))
+    old = make_done_job(client)
+    rerun = client.post(f"/api/jobs/{old}/rerun").json()["id"]
+    wait_status(client, rerun, "done")
+    body = client.post("/api/rebuild-all", data={"force_from": "pitch", "ids": [old]}).json()
+    assert body == {"ids": [], "skipped": [
+        {"id": old, "reason": "no longer the current job for its song"}]}