diff --git a/VERSION b/VERSION index 209e3ef..aabe6ec 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -20 +21 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 532c6df..f893c11 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,3 +1,4 @@ -- Fixed the v19 container startup failure caused by malformed indentation in the Parakeet ONNX loader. -- Preserved automatic download, resume, and offline reuse of the required Parakeet INT8 model snapshot. -- Revalidated both CUDA and CPU Parakeet provider paths with the complete trainer test suite. +- Improved automatic review accuracy for short wake phrases that STT initially hears as similar-sounding words. +- Added a conservative Faster Whisper confirmation pass that uses the currently configured wake phrase only when the unbiased transcript is already phonetically close. +- Kept unconfirmed close transcripts in the manual review inbox instead of allowing them to become harmful negative training samples. +- Added visible guided-transcript and review-reason details, plus retry support for ambiguous clips through Review Now. diff --git a/static/index.html b/static/index.html index d6c08e4..8bff3d4 100644 --- a/static/index.html +++ b/static/index.html @@ -2341,6 +2341,7 @@ if (item.average_probability !== null && item.average_probability !== undefined) meta.push(`avg ${escapeHtml(item.average_probability)}`); if (item.detection_profile) meta.push(`profile ${escapeHtml(formatDetectionProfile(item.detection_profile))}`); if (item.auto_review_status) meta.push(`auto ${escapeHtml(String(item.auto_review_status).replaceAll("_", " "))}`); + if (item.auto_review_match_method === "guided_close_match") meta.push(`guided wake confirmation`); if (item.peak_probability_cutoff !== null && item.peak_probability_cutoff !== undefined) meta.push(`peak cutoff ${escapeHtml(item.peak_probability_cutoff)}`); if (item.probability_cutoff !== null && item.probability_cutoff !== undefined) meta.push(`avg cutoff ${escapeHtml(item.probability_cutoff)}`); if (item.active_window_count !== null && item.active_window_count !== undefined && item.min_active_windows !== null && item.min_active_windows !== undefined) { @@ -2369,6 +2370,8 @@
${meta.join("") || `No metadata attached`}
${item.transcript ? `
STT transcript
${escapeHtml(item.transcript)}
` : ""} + ${item.auto_review_guided_transcript ? `
Guided wake check
${escapeHtml(item.auto_review_guided_transcript)}
` : ""} + ${item.auto_review_reason ? `
${escapeHtml(item.auto_review_reason)}
` : ""} ${item.auto_review_error ? `
Auto review error: ${escapeHtml(item.auto_review_error)}
` : ""}
Stored as ${escapeHtml(item.saved_as)} ยท ${escapeHtml(formatSummary)}
diff --git a/tests/test_auto_train.py b/tests/test_auto_train.py index 9332042..94f0502 100644 --- a/tests/test_auto_train.py +++ b/tests/test_auto_train.py @@ -111,6 +111,21 @@ class AutoTrainTests(unittest.TestCase): self.assertTrue(trainer._transcript_contains_wake_phrase("Okay, HEY TATER!", "hey_tater")) self.assertFalse(trainer._transcript_contains_wake_phrase("Turn on the television", "hey tater")) + def test_phrase_similarity_recognizes_real_short_clip_mishearings(self): + for transcript in ("Hey, haters.", "Hate hater.", "Hey Ganger.", "Hey, gator."): + with self.subTest(transcript=transcript): + self.assertGreaterEqual( + trainer._wake_phrase_similarity(transcript, "hey tater"), + trainer.WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY, + ) + + for transcript in ("turn on the lights", "what is the weather", "play some music"): + with self.subTest(transcript=transcript): + self.assertLess( + trainer._wake_phrase_similarity(transcript, "hey tater"), + trainer.WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY, + ) + def test_stt_engine_selection_uses_managed_models(self): config = trainer._normalize_auto_train_config( { @@ -160,6 +175,39 @@ class AutoTrainTests(unittest.TestCase): faster.assert_called_once() parakeet.assert_called_once() + def test_guided_faster_whisper_uses_dynamic_wake_phrase(self): + fake_model = SimpleNamespace( + transcribe=Mock( + return_value=( + iter([SimpleNamespace(text=" hello "), SimpleNamespace(text="potato ")]), + SimpleNamespace(), + ) + ) + ) + with ( + patch.object( + trainer, + "_resolve_faster_whisper_runtime", + return_value=("cuda", "float16"), + ), + patch.object(trainer, "_load_faster_whisper_model", return_value=fake_model), + ): + transcript = trainer._transcribe_capture_with_faster_whisper_guided( + Path("wake.wav"), + model="small.en", + language="en", + wake_phrase="Hello_Potato", + ) + + self.assertEqual(transcript, "hello potato") + _, kwargs = fake_model.transcribe.call_args + self.assertEqual(kwargs["hotwords"], "hello potato") + self.assertIn("hello potato", kwargs["initial_prompt"]) + self.assertEqual(kwargs["beam_size"], 5) + self.assertEqual(kwargs["best_of"], 5) + self.assertEqual(kwargs["temperature"], 0.0) + self.assertFalse(kwargs["condition_on_previous_text"]) + def test_parakeet_loader_prefers_cuda_then_cpu(self): fake_model = object() fake_onnx_asr = SimpleNamespace(load_model=Mock(return_value=fake_model)) @@ -251,6 +299,7 @@ class AutoTrainTests(unittest.TestCase): self.assertNotIn('id="autoSttModel"', source) self.assertNotIn('id="autoSttDevice"', source) self.assertNotIn('id="autoSttComputeType"', source) + self.assertIn("Guided wake check", source) def test_phrase_miss_moves_wake_trigger_to_negative_samples(self): self.add_capture() @@ -279,6 +328,80 @@ class AutoTrainTests(unittest.TestCase): self.assertEqual(metadata["auto_review_status"], "wake_phrase_detected") self.assertEqual(trainer.AUTO_TRAIN_STATE["pending_negative_count"], 0) + def test_close_transcript_uses_guided_faster_whisper_confirmation(self): + audio_path = self.add_capture() + with ( + patch.object(trainer, "_transcribe_capture", return_value="Hey, haters."), + patch.object( + trainer, + "_transcribe_capture_with_faster_whisper_guided", + return_value="Hey Tater", + ) as guided, + ): + trainer._auto_review_capture("wake.wav") + + self.assertTrue(audio_path.exists()) + self.assertFalse(list(trainer.NEGATIVE_DIR.glob("*.wav"))) + metadata = trainer._load_sidecar_json(audio_path) + self.assertEqual(metadata["auto_review_status"], "wake_phrase_detected") + self.assertEqual(metadata["transcript"], "Hey, haters.") + self.assertEqual(metadata["auto_review_guided_transcript"], "Hey Tater") + self.assertEqual(metadata["auto_review_match_method"], "guided_close_match") + self.assertGreaterEqual( + metadata["auto_review_phrase_similarity"], + trainer.WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY, + ) + guided.assert_called_once() + guided_args, guided_kwargs = guided.call_args + self.assertEqual(guided_args[0].resolve(), audio_path.resolve()) + self.assertEqual( + guided_kwargs, + { + "model": "small.en", + "language": "en", + "wake_phrase": "hey tater", + }, + ) + + def test_unconfirmed_close_transcript_stays_for_manual_review(self): + audio_path = self.add_capture() + with ( + patch.object(trainer, "_transcribe_capture", return_value="Hate hater."), + patch.object( + trainer, + "_transcribe_capture_with_faster_whisper_guided", + return_value="Hate hater.", + ), + ): + trainer._auto_review_capture("wake.wav") + + self.assertTrue(audio_path.exists()) + self.assertFalse(list(trainer.NEGATIVE_DIR.glob("*.wav"))) + metadata = trainer._load_sidecar_json(audio_path) + self.assertEqual(metadata["auto_review_status"], "wake_phrase_ambiguous") + self.assertEqual(metadata["transcript"], "Hate hater.") + self.assertEqual(metadata["auto_review_guided_transcript"], "Hate hater.") + self.assertEqual(trainer.AUTO_TRAIN_STATE["pending_negative_count"], 0) + + self.assertEqual(trainer._queue_pending_auto_reviews(), 0) + self.assertEqual(trainer._queue_pending_auto_reviews(force=True), 1) + + def test_close_parakeet_transcript_stays_for_manual_review(self): + audio_path = self.add_capture() + trainer.AUTO_TRAIN_CONFIG["stt_engine"] = trainer.STT_ENGINE_PARAKEET_ONNX + with ( + patch.object(trainer, "_transcribe_capture", return_value="Hey Ganger."), + patch.object(trainer, "_transcribe_capture_with_faster_whisper_guided") as guided, + ): + trainer._auto_review_capture("wake.wav") + + guided.assert_not_called() + self.assertTrue(audio_path.exists()) + self.assertFalse(list(trainer.NEGATIVE_DIR.glob("*.wav"))) + metadata = trainer._load_sidecar_json(audio_path) + self.assertEqual(metadata["auto_review_status"], "wake_phrase_ambiguous") + self.assertEqual(metadata["auto_review_stt_engine"], "parakeet_onnx") + def test_matching_phrase_is_deleted_when_cleanup_is_enabled(self): audio_path = self.add_capture() trainer.AUTO_TRAIN_CONFIG["delete_confirmed_wakes"] = True diff --git a/trainer_server.py b/trainer_server.py index 09bb665..928ad2e 100644 --- a/trainer_server.py +++ b/trainer_server.py @@ -20,6 +20,7 @@ import unicodedata import wave from array import array from datetime import datetime, timedelta, timezone +from difflib import SequenceMatcher from math import isfinite, log10 from pathlib import Path from typing import Dict, Any, List, Callable, Optional, Tuple @@ -118,6 +119,7 @@ DEFAULT_PARAKEET_ONNX_REPO = os.environ.get( "istupakov/parakeet-tdt-0.6b-v3-onnx", ) DEFAULT_PARAKEET_ONNX_QUANTIZATION = "int8" +WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY = 0.68 AUTO_TRAIN_DEFAULT_CONFIG: Dict[str, Any] = { "enabled": False, @@ -835,6 +837,28 @@ def _transcript_contains_wake_phrase(transcript: Any, wake_phrase: Any) -> bool: return f" {normalized_phrase} " in f" {normalized_transcript} " +def _wake_phrase_similarity(transcript: Any, wake_phrase: Any) -> float: + transcript_words = _normalize_transcript_text(transcript).split() + phrase_words = _normalize_transcript_text(wake_phrase).split() + if not transcript_words or not phrase_words: + return 0.0 + if _transcript_contains_wake_phrase(transcript, wake_phrase): + return 1.0 + + phrase_token = "".join(phrase_words) + minimum_words = max(1, len(phrase_words) - 1) + maximum_words = min(len(transcript_words), len(phrase_words) + 1) + best_score = 0.0 + for word_count in range(minimum_words, maximum_words + 1): + for start in range(0, len(transcript_words) - word_count + 1): + candidate = "".join(transcript_words[start : start + word_count]) + best_score = max( + best_score, + SequenceMatcher(None, candidate, phrase_token).ratio(), + ) + return best_score + + def _captured_event_is_close_miss(metadata: Dict[str, Any]) -> bool: event_type = str(metadata.get("event_type") or "captured").strip().lower() return "close" in event_type @@ -925,6 +949,40 @@ def _transcribe_capture_with_faster_whisper(audio_path: Path, *, model: str, lan return transcript +def _transcribe_capture_with_faster_whisper_guided( + audio_path: Path, + *, + model: str, + language: str, + wake_phrase: str, +) -> str: + normalized_phrase = _normalize_transcript_text(wake_phrase) + if not normalized_phrase: + return "" + device, compute_type = _resolve_faster_whisper_runtime("auto", "auto") + whisper_model = _load_faster_whisper_model( + model_name=model, + device=device, + compute_type=compute_type, + ) + with FASTER_WHISPER_TRANSCRIBE_LOCK: + segments, _info = whisper_model.transcribe( + str(audio_path), + language=language or None, + beam_size=5, + best_of=5, + temperature=0.0, + condition_on_previous_text=False, + initial_prompt=f'The wake phrase is "{normalized_phrase}".', + hotwords=normalized_phrase, + ) + return re.sub( + r"\s+", + " ", + " ".join(str(segment.text or "").strip() for segment in segments), + ).strip() + + def _parakeet_onnx_providers() -> List[str]: try: import onnxruntime as ort @@ -1119,7 +1177,7 @@ def _queue_pending_auto_reviews(*, force: bool = False) -> int: metadata.pop("auto_review_status", None) _write_sidecar_json(audio_path, metadata) status = "" - if force and status in {"error", "no_speech"}: + if force and status in {"error", "no_speech", "wake_phrase_ambiguous"}: metadata.pop("auto_review_status", None) _write_sidecar_json(audio_path, metadata) status = "" @@ -1220,11 +1278,38 @@ def _auto_review_capture(file_name: str) -> None: _record_auto_review_result(file_name=file_name, transcript=transcript, result="no_speech") return - if _transcript_contains_wake_phrase(transcript, wake_phrase): + phrase_similarity = _wake_phrase_similarity(transcript, wake_phrase) + phrase_detected = _transcript_contains_wake_phrase(transcript, wake_phrase) + match_method = "exact" if phrase_detected else "" + metadata["auto_review_phrase_similarity"] = round(phrase_similarity, 4) + + if ( + not phrase_detected + and phrase_similarity >= WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY + and stt_engine == STT_ENGINE_FASTER_WHISPER + ): + guided_transcript = _transcribe_capture_with_faster_whisper_guided( + audio_path, + model=str(metadata["auto_review_stt_model"]), + language=str(config.get("language") or DEFAULT_LANGUAGE), + wake_phrase=wake_phrase, + ) + metadata["auto_review_guided_transcript"] = guided_transcript + if _transcript_contains_wake_phrase(guided_transcript, wake_phrase): + phrase_detected = True + match_method = "guided_close_match" + + if match_method: + metadata["auto_review_match_method"] = match_method + + if phrase_detected: + guided_confirmation = match_method == "guided_close_match" if is_close_miss: metadata["auto_review_status"] = "approved_positive" metadata["auto_review_reason"] = ( - "Close miss contained the configured wake phrase and was promoted to a positive sample." + "Close miss was confirmed as the configured wake phrase and promoted to a positive sample." + if guided_confirmation + else "Close miss contained the configured wake phrase and was promoted to a positive sample." ) metadata["auto_positive"] = True _write_sidecar_json(audio_path, metadata) @@ -1249,11 +1334,29 @@ def _auto_review_capture(file_name: str) -> None: ) return metadata["auto_review_status"] = "wake_phrase_detected" - metadata["auto_review_reason"] = "Wake phrase found in transcript; left for manual positive review." + metadata["auto_review_reason"] = ( + "Wake phrase confirmed by a guided second STT pass; left for manual positive review." + if guided_confirmation + else "Wake phrase found in transcript; left for manual positive review." + ) _write_sidecar_json(audio_path, metadata) _record_auto_review_result(file_name=file_name, transcript=transcript, result="wake_phrase_detected") return + if phrase_similarity >= WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY: + metadata["auto_review_status"] = "wake_phrase_ambiguous" + metadata["auto_review_reason"] = ( + "STT sounded close to the configured wake phrase but could not confirm it; " + "left for manual review." + ) + _write_sidecar_json(audio_path, metadata) + _record_auto_review_result( + file_name=file_name, + transcript=transcript, + result="wake_phrase_ambiguous", + ) + return + if is_close_miss: metadata["auto_review_status"] = "close_miss_phrase_not_detected" metadata["auto_review_reason"] = ( @@ -2162,6 +2265,9 @@ def _captured_item_from_path(audio_path: Path) -> Dict[str, Any]: "auto_review_status": meta.get("auto_review_status") or "", "auto_review_reason": meta.get("auto_review_reason") or "", "auto_review_error": meta.get("auto_review_error") or "", + "auto_review_guided_transcript": meta.get("auto_review_guided_transcript") or "", + "auto_review_phrase_similarity": meta.get("auto_review_phrase_similarity"), + "auto_review_match_method": meta.get("auto_review_match_method") or "", "size_bytes": stat.st_size, "audio_url": f"/api/audio/captured/{audio_path.name}", }