webui: add REBUILD ALL FROM… to rebuild every song from a stage
When an update improves a stage, existing songs only pick it up when each is rebuilt from that stage. POST /api/rebuild-all queues an ordinary rebuild with unchanged inputs for the current job of every song: jobs that were themselves rebuilt are skipped, as are songs still in progress and songs whose upload is gone when the stage is acquire. The sidebar button opens a dialog to pick the stage and confirm the count.
This commit is contained in:
@@ -177,6 +177,11 @@ to redo a stage anyway — pick `package` to repackage without re-downloading.
|
||||
It can only widen what is rerun, so an edit that invalidates an earlier stage
|
||||
still wins.
|
||||
|
||||
**REBUILD ALL FROM…**, under CREATE SONG, does the same for every song at once:
|
||||
pick the stage an update improved and each song is rebuilt from there with the
|
||||
inputs it has now. Songs still in progress are skipped, as are songs whose
|
||||
uploaded video is gone when the stage is `acquire`.
|
||||
|
||||
## YouTube download failures
|
||||
|
||||
`HTTP Error 403: Forbidden` from the acquire stage means YouTube served the
|
||||
|
||||
@@ -113,6 +113,31 @@ body {
|
||||
#sidebar .eyebrow, #jobs .empty-note { padding: 0 18px; }
|
||||
#create-song { display: block; width: calc(100% - 36px); margin: 0 18px 20px; padding: 9px 12px; }
|
||||
|
||||
#rebuild-all { display: block; margin: -8px 18px 20px; }
|
||||
#rebuild-all-note { margin: -12px 18px 20px; }
|
||||
|
||||
#rebuild-all-dialog {
|
||||
max-width: 420px;
|
||||
padding: 26px 28px;
|
||||
color: var(--text);
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
#rebuild-all-dialog::backdrop { background: rgb(10 10 15 / 0.75); }
|
||||
#rebuild-all-dialog h2 {
|
||||
margin: 0 0 14px;
|
||||
font-family: var(--font-display);
|
||||
font-weight: 700;
|
||||
font-size: 0.8rem;
|
||||
letter-spacing: 0.22em;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
#rebuild-all-form { display: grid; gap: 16px; }
|
||||
#rebuild-all-form p { margin: 0; font-size: 0.88rem; line-height: 1.5; }
|
||||
#rebuild-all-count { color: var(--warn); }
|
||||
#rebuild-all-error { min-height: 1.2em; color: var(--fail); }
|
||||
#rebuild-all-error:empty { display: none; }
|
||||
|
||||
#jobs { display: flex; flex-direction: column; gap: 1px; margin-top: 8px; }
|
||||
|
||||
.job {
|
||||
@@ -518,11 +543,31 @@ button { cursor: pointer; }
|
||||
</div>
|
||||
<div>
|
||||
<button type="button" id="create-song" class="primary">CREATE SONG</button>
|
||||
<button type="button" id="rebuild-all" class="action">REBUILD ALL FROM…</button>
|
||||
<p id="rebuild-all-note" class="empty-note" role="status" hidden></p>
|
||||
<h2 class="eyebrow">QUEUE</h2>
|
||||
<div id="jobs"></div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
<dialog id="rebuild-all-dialog" aria-labelledby="rebuild-all-heading">
|
||||
<form id="rebuild-all-form" method="dialog">
|
||||
<h2 id="rebuild-all-heading">REBUILD ALL SONGS</h2>
|
||||
<p>Reruns every song from the chosen stage onwards with the inputs it has
|
||||
now, for when a songtrace update improves that stage.</p>
|
||||
<label class="field">
|
||||
<span>Restart from</span>
|
||||
<select name="force_from" required></select>
|
||||
</label>
|
||||
<p id="rebuild-all-count"></p>
|
||||
<p id="rebuild-all-error" role="alert"></p>
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="primary">REBUILD ALL</button>
|
||||
<button type="button" class="action" id="rebuild-all-cancel">CANCEL</button>
|
||||
</div>
|
||||
</form>
|
||||
</dialog>
|
||||
|
||||
<main id="main">
|
||||
<section id="new-song">
|
||||
<h2>NEW SONG</h2>
|
||||
@@ -702,6 +747,62 @@ createSong.addEventListener("click", () => {
|
||||
form.elements.title.focus();
|
||||
});
|
||||
|
||||
const rebuildAllButton = document.getElementById("rebuild-all");
|
||||
const rebuildAllNote = document.getElementById("rebuild-all-note");
|
||||
const rebuildAllDialog = document.getElementById("rebuild-all-dialog");
|
||||
const rebuildAllForm = document.getElementById("rebuild-all-form");
|
||||
const rebuildAllCount = document.getElementById("rebuild-all-count");
|
||||
const rebuildAllError = document.getElementById("rebuild-all-error");
|
||||
|
||||
// The server's rule, for the count only: a rebuilt job's rebuild is the song
|
||||
// now, and two submissions of one song share it.
|
||||
function currentSongs() {
|
||||
const rebuilt = new Set(JOBS.map(j => j.rebuilt_from));
|
||||
const songs = new Map();
|
||||
for (const job of [...JOBS].reverse()) {
|
||||
if (!rebuilt.has(job.id)) songs.set(`${job.artist} - ${job.title}`, job);
|
||||
}
|
||||
return [...songs.values()];
|
||||
}
|
||||
|
||||
rebuildAllButton.addEventListener("click", () => {
|
||||
const songs = currentSongs();
|
||||
const idle = songs.filter(j => j.status === "done" || j.status === "failed").length;
|
||||
rebuildAllCount.textContent = idle === 1 ? "1 song will be rebuilt."
|
||||
: `${idle} songs will be rebuilt.`;
|
||||
if (idle < songs.length) {
|
||||
rebuildAllCount.textContent += ` ${songs.length - idle} still in progress will be skipped.`;
|
||||
}
|
||||
rebuildAllError.textContent = "";
|
||||
rebuildAllForm.reset();
|
||||
rebuildAllDialog.showModal();
|
||||
});
|
||||
|
||||
document.getElementById("rebuild-all-cancel").addEventListener(
|
||||
"click", () => rebuildAllDialog.close());
|
||||
|
||||
rebuildAllForm.addEventListener("submit", async (e) => {
|
||||
e.preventDefault();
|
||||
const stage = rebuildAllForm.elements.force_from.value;
|
||||
const resp = await fetch("/api/rebuild-all", {
|
||||
method: "POST", body: new FormData(rebuildAllForm) });
|
||||
if (!resp.ok) {
|
||||
const body = await resp.json().catch(() => ({}));
|
||||
rebuildAllError.textContent = body.detail || `error: ${resp.status}`;
|
||||
return;
|
||||
}
|
||||
const { ids, skipped } = await resp.json();
|
||||
rebuildAllDialog.close();
|
||||
let note = `Queued ${ids.length} rebuild${ids.length === 1 ? "" : "s"} from ${stage}.`;
|
||||
if (skipped.length) {
|
||||
const reasons = [...new Set(skipped.map(s => s.reason))].join("; ");
|
||||
note += ` Skipped ${skipped.length}: ${reasons}.`;
|
||||
}
|
||||
rebuildAllNote.textContent = note;
|
||||
rebuildAllNote.hidden = false;
|
||||
refreshJobs();
|
||||
});
|
||||
|
||||
function jobCard(job) {
|
||||
const card = document.createElement("button");
|
||||
card.type = "button";
|
||||
@@ -780,6 +881,8 @@ async function loadStages() {
|
||||
const select = form.elements.force_from;
|
||||
select.replaceChildren(new Option("only what changed", ""),
|
||||
...STAGES.map(s => new Option(s, s)));
|
||||
rebuildAllForm.elements.force_from.replaceChildren(
|
||||
new Option("choose a stage", ""), ...STAGES.map(s => new Option(s, s)));
|
||||
}
|
||||
|
||||
function renderStageTrack(job) {
|
||||
|
||||
@@ -329,6 +329,32 @@ def create_app(cfg: Config, runner=run_pipeline) -> FastAPI:
|
||||
rebuild_policy.plan(parent.inp, inp), force_from))
|
||||
return _enqueue(new_id, inp, rebuilt_from=job_id, force_stage=force_from)
|
||||
|
||||
@app.post("/api/rebuild-all", status_code=201)
|
||||
def rebuild_all(force_from: str = Form(...)):
|
||||
"""Rebuild every song from `force_from` on, for when songtrace itself
|
||||
improved that stage. Each is an ordinary rebuild with unchanged inputs."""
|
||||
if force_from not in STAGE_ORDER:
|
||||
raise HTTPException(400, f"unknown stage {force_from!r}")
|
||||
with lock:
|
||||
# a job that was rebuilt is history: its rebuild is the song now,
|
||||
# possibly under another slug
|
||||
rebuilt = {j.rebuilt_from for j in jobs.values()}
|
||||
latest = {song_slug(j.inp): j for j in (jobs[i] for i in order)
|
||||
if j.id not in rebuilt}
|
||||
ids, skipped = [], []
|
||||
for job in latest.values():
|
||||
if job.status in ("queued", "running"):
|
||||
skipped.append({"id": job.id, "reason": "still in progress"})
|
||||
continue
|
||||
try:
|
||||
_require_source(job.inp, job.inp, force_from)
|
||||
except HTTPException as e:
|
||||
skipped.append({"id": job.id, "reason": e.detail})
|
||||
continue
|
||||
ids.append(_enqueue(uuid.uuid4().hex, job.inp, rebuilt_from=job.id,
|
||||
force_stage=force_from)["id"])
|
||||
return {"ids": ids, "skipped": skipped}
|
||||
|
||||
@app.get("/api/stages")
|
||||
def list_stages():
|
||||
return {"stages": list(STAGE_ORDER)}
|
||||
|
||||
+74
-1
@@ -834,7 +834,7 @@ def test_narrow_stage_labels_stay_in_the_accessibility_tree():
|
||||
|
||||
def test_rebuild_mode_reuses_the_one_form():
|
||||
html = _index_html()
|
||||
assert html.count("<form") == 1
|
||||
assert html.count('name="lyrics"') == 1 # one song form, not a rebuild copy
|
||||
assert _rebuild_flaws(html) == []
|
||||
|
||||
|
||||
@@ -1746,3 +1746,76 @@ def test_submitting_crlf_lyrics_stores_them_with_lf(tmp_path):
|
||||
client, _ = make_client(tmp_path, ok_runner)
|
||||
job_id = make_done_job(client, lyrics="one\r\ntwo")
|
||||
assert client.get(f"/api/jobs/{job_id}/input").json()["lyrics"] == "one\ntwo"
|
||||
|
||||
|
||||
def test_rebuild_all_rebuilds_every_song_from_the_stage(tmp_path):
|
||||
seen = []
|
||||
client, _ = make_client(tmp_path, capture_runner_factory(seen))
|
||||
first = make_done_job(client, title="One")
|
||||
second = make_done_job(client, title="Two")
|
||||
resp = client.post("/api/rebuild-all", data={"force_from": "pitch"})
|
||||
assert resp.status_code == 201
|
||||
body = resp.json()
|
||||
assert len(body["ids"]) == 2 and body["skipped"] == []
|
||||
jobs = [wait_status(client, i, "done") for i in body["ids"]]
|
||||
assert {j["rebuilt_from"] for j in jobs} == {first, second}
|
||||
assert [s["force_from"] for s in seen[2:]] == ["pitch", "pitch"]
|
||||
assert [s["inp"] for s in seen[2:]] == [s["inp"] for s in seen[:2]]
|
||||
|
||||
|
||||
def test_rebuild_all_skips_jobs_that_were_already_rebuilt(tmp_path):
|
||||
client, _ = make_client(tmp_path, capture_runner_factory([]))
|
||||
parent = make_done_job(client)
|
||||
renamed = client.post(f"/api/jobs/{parent}/rebuild",
|
||||
data=rebuild_form(title="Renamed")).json()["id"]
|
||||
wait_status(client, renamed, "done")
|
||||
body = client.post("/api/rebuild-all", data={"force_from": "notes"}).json()
|
||||
assert len(body["ids"]) == 1
|
||||
job = wait_status(client, body["ids"][0], "done")
|
||||
assert job["rebuilt_from"] == renamed
|
||||
|
||||
|
||||
def test_rebuild_all_takes_the_latest_job_for_a_song(tmp_path):
|
||||
client, _ = make_client(tmp_path, capture_runner_factory([]))
|
||||
make_done_job(client)
|
||||
latest = make_done_job(client)
|
||||
body = client.post("/api/rebuild-all", data={"force_from": "grid"}).json()
|
||||
assert len(body["ids"]) == 1
|
||||
assert wait_status(client, body["ids"][0], "done")["rebuilt_from"] == latest
|
||||
|
||||
|
||||
def test_rebuild_all_skips_songs_in_progress(tmp_path):
|
||||
gate = threading.Event()
|
||||
|
||||
def blocking_runner(inp, cfg, log=print, force_from=None, stages=None):
|
||||
gate.wait(timeout=5)
|
||||
return PipelineResult(output_dir=cfg.output_dir / song_slug(inp))
|
||||
|
||||
client, _ = make_client(tmp_path, blocking_runner)
|
||||
try:
|
||||
busy = client.post("/api/jobs", data={
|
||||
"title": "T", "artist": "A", "lyrics": "x", "youtube_url": "http://y"}).json()["id"]
|
||||
body = client.post("/api/rebuild-all", data={"force_from": "pitch"}).json()
|
||||
assert body == {"ids": [], "skipped": [{"id": busy, "reason": "still in progress"}]}
|
||||
finally:
|
||||
gate.set()
|
||||
|
||||
|
||||
def test_rebuild_all_from_acquire_skips_a_song_whose_upload_is_gone(tmp_path):
|
||||
client, _ = make_client(tmp_path, capture_runner_factory([]))
|
||||
job_id = client.post("/api/jobs", data={"title": "T", "artist": "A", "lyrics": "x"},
|
||||
files={"video": ("v.mp4", io.BytesIO(b"data"), "video/mp4")}
|
||||
).json()["id"]
|
||||
wait_status(client, job_id, "done")
|
||||
shutil.rmtree(tmp_path / "work" / "uploads", ignore_errors=True)
|
||||
body = client.post("/api/rebuild-all", data={"force_from": "acquire"}).json()
|
||||
assert body["ids"] == []
|
||||
assert body["skipped"][0]["id"] == job_id
|
||||
assert "upload" in body["skipped"][0]["reason"]
|
||||
|
||||
|
||||
def test_rebuild_all_rejects_an_unknown_stage(tmp_path):
|
||||
client, _ = make_client(tmp_path, ok_runner)
|
||||
make_done_job(client)
|
||||
assert client.post("/api/rebuild-all", data={"force_from": "nope"}).status_code == 400
|
||||
assert client.post("/api/rebuild-all", data={}).status_code == 422
|
||||
|
||||
Reference in New Issue
Block a user