transcribe: scope missing_spans to the unmatched audio itself, not the whole vocal section

Also fix a secondary ripple in build_notes(): a skipped row (inside a
missing span) no longer advances prev_line/prev_end, so it can't corrupt
line-initial spacing or break timing for its own line's later words.
This commit is contained in:
2026-07-28 00:48:19 -07:00
parent b3b9b9ee4b
commit b506de6147
5 changed files with 67 additions and 21 deletions
+9 -3
View File
@@ -84,14 +84,20 @@ def build_notes(rows: list[dict], times, f0, conf, language: str | None,
) -> tuple[list[TimedNote], list[float], list[str]]:
notes: list[TimedNote] = []
breaks: list[float] = []
prev_line, prev_end = None, 0.0
prev_line, prev_end, last_seen_line = None, 0.0, None
for row in rows:
if prev_line is not None and row["line_index"] > prev_line:
# last_seen_line tracks every row (skipped or not) so a run of
# skipped rows can't trigger a duplicate break on the next row of
# the same still-unseen line; prev_line/prev_end only advance on
# rows that actually emit notes, so a skipped row can't corrupt
# line_initial or break timing for its own line's later words.
if last_seen_line is not None and row["line_index"] > last_seen_line:
breaks.append((prev_end + row["start"]) / 2)
last_seen_line = row["line_index"]
line_initial = row["line_index"] != prev_line
prev_line, prev_end = row["line_index"], row["end"]
if any(a <= row["start"] < b for a, b in missing_spans):
continue
prev_line, prev_end = row["line_index"], row["end"]
segs = segment_span(times, f0, conf, row["start"], row["end"])
if not segs:
pitch = notes[-1].pitch if notes else 0
+1 -1
View File
@@ -106,7 +106,7 @@ def _run(ctx: Ctx) -> None:
"wrong lyrics or wrong recording?")
# normalised so an inverted span can never collapse a section onto one instant
sections = [(min(a, b), max(a, b)) for a, b in vocal_sections(segments)]
dropped_lines, missing_spans = classify_lyrics_match(segments, ref, sections)
dropped_lines, missing_spans = classify_lyrics_match(segments, ref)
matched = [i for i in range(len(ref.lines)) if i not in set(dropped_lines)]
vocals = ctx.work / "vocals.wav"
rows = []
+22 -11
View File
@@ -39,14 +39,17 @@ def lyrics_similarity(segments, ref: RefLyrics) -> float:
return matcher.ratio()
def classify_lyrics_match(segments, ref: RefLyrics, sections
def classify_lyrics_match(segments, ref: RefLyrics, pad: float = 1.0
) -> tuple[list[int], list[tuple[float, float]]]:
"""Split reference lines into matched vs. dropped, and flag sung spans no
reference line explains.
dropped_lines: reference line indices covered only by 'insert' opcodes
(present in the lyrics, never heard). missing_spans: vocal `sections`
containing 'delete' opcodes (heard, but matching no reference word).
(present in the lyrics, never heard). missing_spans: the padded, merged
bounds of the heard segments behind 'delete' opcodes (heard, but matching
no reference word) — scoped tightly to the unmatched audio itself, not
the whole vocal section it falls in, so matched lines sharing a section
with an ad-lib keep their notes.
"""
matcher, seg_of, line_of = match_lyrics(segments, ref)
covered = [False] * len(line_of)
@@ -59,17 +62,25 @@ def classify_lyrics_match(segments, ref: RefLyrics, sections
by_line.setdefault(li, []).append(covered[j])
dropped_lines = sorted(li for li, cov in by_line.items() if not any(cov))
missing = set()
spans = []
for tag, ai1, ai2, _, _ in matcher.get_opcodes():
if tag != "delete":
continue
si = seg_of[ai1]
mid = (segments[si].start + segments[si].end) / 2
for a, b in sections:
if a <= mid <= b:
missing.add((a, b))
break
return dropped_lines, sorted(missing)
sis = [seg_of[i] for i in range(ai1, ai2)]
start = max(0.0, min(segments[si].start for si in sis) - pad)
end = max(segments[si].end for si in sis) + pad
spans.append((start, end))
return dropped_lines, _merge_spans(spans)
def _merge_spans(spans: list[tuple[float, float]]) -> list[tuple[float, float]]:
merged: list[list[float]] = []
for a, b in sorted(spans):
if merged and a <= merged[-1][1]:
merged[-1][1] = max(merged[-1][1], b)
else:
merged.append([a, b])
return [tuple(s) for s in merged]
def _run(ctx: Ctx) -> None:
+17
View File
@@ -325,6 +325,23 @@ def test_warnings_report_missing_spans():
assert warnings == ["missing lyrics near 1:21–1:29"]
def test_build_notes_skipped_row_mid_line_does_not_corrupt_spacing_or_breaks():
# "gone" falls inside a missing span and is skipped; "away" is the same
# line's next word and must still get its leading space, and the break
# before the next line must use the last *emitted* word's end (3.4, from
# "away"), not the skipped row's end (2.4).
times, f0, conf = flat_pitch()
rows = [
{"word": "keep", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 0},
{"word": "gone", "start": 2.0, "end": 2.4, "score": 0.9, "line_index": 0},
{"word": "away", "start": 3.0, "end": 3.4, "score": 0.9, "line_index": 0},
{"word": "next", "start": 5.0, "end": 5.4, "score": 0.9, "line_index": 1},
]
notes, breaks, _ = build_notes(rows, times, f0, conf, "English",
missing_spans=[(1.9, 2.5)])
assert [n.text for n in notes] == ["keep", " away", "next"]
assert breaks == [(3.4 + 5.0) / 2]
def test_notes_carry_line_index():
times, f0, conf = flat_pitch()
rows = [{"word": "hey", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 3}]
+18 -6
View File
@@ -30,8 +30,7 @@ def test_similarity_low_for_wrong_lyrics():
def test_classify_lyrics_match_flags_extra_reference_line():
ref = normalize_lyrics("she's got a smile\nthat it seems to me\nbut then it fades")
segs = [seg(1, 4, "she's got a smile that it seems to me")]
sections = vocal_sections(segs)
dropped, missing = classify_lyrics_match(segs, ref, sections)
dropped, missing = classify_lyrics_match(segs, ref)
assert dropped == [2]
assert missing == []
@@ -39,8 +38,7 @@ def test_classify_lyrics_match_flags_unmatched_singing():
ref = normalize_lyrics("she's got a smile\nthat it seems to me")
segs = [seg(1, 4, "she's got a smile that it seems to me"),
seg(20, 23, "completely unrelated ad lib bridge")]
sections = vocal_sections(segs)
dropped, missing = classify_lyrics_match(segs, ref, sections)
dropped, missing = classify_lyrics_match(segs, ref)
assert dropped == []
assert missing == [(19.0, 24.0)]
@@ -49,6 +47,20 @@ def test_classify_lyrics_match_keeps_partially_matched_lines():
# line 1 heard with one word garbled ("it" -> "bit"): still a replace/equal
# match, so the line stays matched rather than dropped
segs = [seg(1, 4, "she's got a smile that bit seems to me")]
sections = vocal_sections(segs)
dropped, _ = classify_lyrics_match(segs, ref, sections)
dropped, _ = classify_lyrics_match(segs, ref)
assert dropped == []
def test_classify_lyrics_match_scopes_missing_span_to_adlib_not_whole_section():
# matched line and an unmatched ad-lib close enough together that
# vocal_sections would merge them into one section; missing_spans must
# stay tight around the ad-lib so the matched line keeps its notes.
ref = normalize_lyrics("she's got a smile\nthat it seems to me")
segs = [seg(1, 4, "she's got a smile that it seems to me"),
seg(6, 8, "completely unrelated ad lib")]
sections = vocal_sections(segs)
assert sections == [(0.0, 9.0)] # confirms the two segments do merge
dropped, missing = classify_lyrics_match(segs, ref)
assert dropped == []
assert missing == [(5.0, 9.0)]
for a, b in missing:
assert not (a <= 1.0 < b) # matched line's start (1.0) untouched