Merge pull request 'notes: judge low confidence against the song, and flag unsingable words' (#28) from relative-confidence into main

Reviewed-on: #28
Reviewed-by: branderson <brad@branderson.io>

src/songtrace/notes.py

@@ -7,7 +7,17 @@ import pyphen
 from .languages import LANGUAGES
 
 CONF_MASK = 0.5          # pitch frames below this are unvoiced
-LOW_SCORE = 0.5          # word alignment score below this -> low_confidence
+# A word is low-confidence when it scores below this share of the song's
+# median alignment score: the scale differs a lot from song to song, so no
+# fixed cutoff holds across them.
+LOW_SCORE_OF_MEDIAN = 0.25
+# ...or when no one sings a word that fast, or it was stretched over more than
+# it can be holding.
+MIN_WORD_SEC = 0.08
+MAX_WORD_SEC = 3.0
+# Flagged notes closer than this share one warning, so the list reads as
+# places to look rather than one line per note.
+WARN_GAP_SEC = 2.0
 SPLIT_SEMITONES = 0.8
 MIN_SEG_SEC = 0.05
 MIN_NOTE_SEC = 0.05      # alignment may hand us zero-length words
@@ -84,6 +94,7 @@ def build_notes(rows: list[dict], times, f0, conf, language: str | None,
                 ) -> tuple[list[TimedNote], list[float], list[str]]:
     notes: list[TimedNote] = []
     breaks: list[float] = []
+    floor = LOW_SCORE_OF_MEDIAN * float(np.median([r["score"] for r in rows])) if rows else 0.0
     prev_line, prev_end = None, 0.0
     for row in rows:
         if prev_line is not None and row["line_index"] > prev_line:
@@ -97,7 +108,8 @@ def build_notes(rows: list[dict], times, f0, conf, language: str | None,
                                _text(row["word"], 0, line_initial), True,
                                row["line_index"]))
             continue
-        low = row["score"] < LOW_SCORE
+        low = (row["score"] < floor
+               or not MIN_WORD_SEC <= row["end"] - row["start"] <= MAX_WORD_SEC)
         sylls = syllabify(row["word"], language)
         if len(segs) < len(sylls):
             segs = _subdivide(segs, len(sylls))
@@ -160,16 +172,19 @@ def _fmt(t: float) -> str:
 
 def _warnings(notes: list[TimedNote], dropped_lines=(), missing_spans=(),
              ref_lines=()) -> list[str]:
-    warnings, run = [], []
-    for n in list(notes) + [None]:
-        if n is not None and n.low_confidence:
-            run.append(n)
-        elif run:
-            first, last = run[0], run[-1]
-            warnings.append(
-                f"{len(run)} low-confidence note{'' if len(run) == 1 else 's'} "
-                f"near {_fmt(first.start)}–{_fmt(last.start + last.duration)}")
-            run = []
+    runs: list[list[TimedNote]] = []
+    for n in notes:
+        if not n.low_confidence:
+            continue
+        last = runs[-1][-1] if runs else None
+        if last is not None and n.start - (last.start + last.duration) < WARN_GAP_SEC:
+            runs[-1].append(n)
+        else:
+            runs.append([n])
+    warnings = [
+        f"{len(run)} low-confidence note{'' if len(run) == 1 else 's'} "
+        f"near {_fmt(run[0].start)}–{_fmt(run[-1].start + run[-1].duration)}"
+        for run in runs]
     for group in _consecutive_runs(dropped_lines):
         n = len(group)
         text = " ".join(ref_lines[group[0]]) if group[0] < len(ref_lines) else ""

tests/test_notes.py

@@ -4,7 +4,7 @@ import numpy as np
 import pytest
 from songtrace.config import Config
 from songtrace.notes import (
-    MIN_NOTE_SEC, TimedNote, build_notes, hz_to_ultrastar, segment_span,
+    MIN_NOTE_SEC, TimedNote, _warnings, build_notes, hz_to_ultrastar, segment_span,
     syllabify,
 )
 from songtrace.pipeline import Ctx, SongInput
@@ -127,11 +127,36 @@ def test_build_notes_unvoiced_word_flagged():
     assert warnings[0] == "1 low-confidence note near 0:01–0:01"
 
 
-def test_build_notes_low_score_word_flagged():
+def _scored_rows(scores, dur=0.4):
+    return [{"word": w, "start": 1.0 + i, "end": 1.0 + i + dur, "score": sc,
+             "line_index": 0}
+            for i, (w, sc) in enumerate(zip(["one", "two", "three", "four", "five"], scores))]
+
+
+def test_build_notes_flags_a_word_scoring_far_below_the_rest():
     times, f0, conf = flat_pitch()
-    rows = [{"word": "hey", "start": 1.0, "end": 1.4, "score": 0.1, "line_index": 0}]
-    notes, _, warnings = build_notes(rows, times, f0, conf, "English")
-    assert all(n.low_confidence for n in notes) and len(warnings) == 1
+    notes, _, warnings = build_notes(_scored_rows([0.4, 0.4, 0.05, 0.4, 0.4]),
+                                     times, f0, conf, "English")
+    assert [n.low_confidence for n in notes] == [False, False, True, False, False]
+    assert len(warnings) == 1
+
+
+def test_build_notes_judges_scores_against_the_song_not_a_fixed_cutoff():
+    # sung vocals score low throughout; uniformly low is not a reason to flag
+    times, f0, conf = flat_pitch()
+    notes, _, warnings = build_notes(_scored_rows([0.05] * 5), times, f0, conf, "English")
+    assert not any(n.low_confidence for n in notes) and warnings == []
+
+
+def test_build_notes_flags_words_too_short_or_too_long_to_be_sung():
+    times, f0, conf = flat_pitch()
+    for dur in (0.05, 3.5):
+        rows = _scored_rows([0.4] * 3)
+        rows[1]["end"] = rows[1]["start"] + dur
+        rows[2]["start"], rows[2]["end"] = rows[1]["end"] + 0.1, rows[1]["end"] + 0.5
+        notes, _, _ = build_notes(rows, times, f0, conf, "English")
+        # flat pitch: one note per word
+        assert [n.low_confidence for n in notes] == [False, True, False], dur
 
 
 def test_build_notes_unvoiced_word_inherits_previous_pitch():
@@ -334,3 +359,13 @@ def test_notes_carry_line_index():
     rows = [{"word": "hey", "start": 1.0, "end": 1.4, "score": 0.9, "line_index": 3}]
     notes, _, _ = build_notes(rows, times, f0, conf, "English")
     assert notes[0].line_index == 3
+
+
+def test_warnings_group_flagged_notes_close_together():
+    def note(start, low):
+        return TimedNote(start, 0.3, 0, "la", low, 0)
+    notes = [note(10.0, True), note(10.5, False), note(11.0, True),   # 0.7s apart
+             note(20.0, True)]                                       # far from both
+    assert _warnings(notes) == [
+        "2 low-confidence notes near 0:10–0:11",
+        "1 low-confidence note near 0:20–0:20"]