mirror of
https://github.com/TaterTotterson/microWakeWord-Trainer-Nvidia-Docker.git
synced 2026-08-12 16:05:34 -06:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2b1320f1f3 | ||
|
|
19ee63a65b | ||
|
|
518df63161 | ||
|
|
6ee228e8d3 |
@@ -171,7 +171,7 @@ Starting a new session does not clear samples. Use the clear buttons in `Samples
|
||||
|
||||
For each new wake-trigger clip sent to the trainer:
|
||||
|
||||
1. Faster Whisper transcribes the audio locally.
|
||||
1. The selected local STT engine transcribes the audio.
|
||||
2. If the transcript contains the configured wake phrase, the clip stays in `Captured Audio` for manual review by default.
|
||||
3. If speech was transcribed but the wake phrase is absent, the clip moves to `/data/negative_samples/` as an auto-reviewed hard negative.
|
||||
4. Empty transcripts, VAD-blocked captures, and captures for another wake word stay out of the automatic negative path.
|
||||
@@ -183,7 +183,7 @@ Two optional cleanup rules are available:
|
||||
|
||||
A close miss with an empty transcript or without the configured phrase stays in `Captured Audio`; it is never turned into a negative automatically. Saving Auto Training settings also scans existing eligible captures. Enabling close-miss promotion reviews previous unreviewed close misses, while enabling cleanup removes previously confirmed good wakes without transcribing them a second time.
|
||||
|
||||
The default `small.en` model uses CUDA with `float16` when CTranslate2 can see an NVIDIA GPU, and falls back to CPU with `int8`. Choose a multilingual Faster Whisper model such as `small` when the wake phrase is not English. Downloaded STT models are cached in `/data/auto_train_models/`.
|
||||
Auto Training exposes only an engine selector. Faster Whisper is the recommended default and uses CUDA with `float16` when CTranslate2 can see an NVIDIA GPU, with a CPU `int8` fallback. The trainer manages `small.en` for English and `small` for other languages. Parakeet ONNX uses the managed INT8 `nemo-parakeet-tdt-0.6b-v3` model with CUDA and CPU fallback. Downloaded STT models are cached in `/data/auto_train_models/`.
|
||||
|
||||
Scheduled training runs only after the configured number of new automatic negatives has accumulated. A successful run securely publishes the trained wake-word name and JSON URL to the linked Tater instance. Tater saves it as the global satellite wake word and pushes the updated setting to every connected satellite, so no satellite firmware change is required.
|
||||
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
- Fixed the training-status endpoint crashing after a training log was created because its log-tail limits were missing.
|
||||
- Restored bounded, incremental training-log updates so the UI can continue showing live progress without repeatedly reading the entire log.
|
||||
- Improved automatic review accuracy for short wake phrases that STT initially hears as similar-sounding words.
|
||||
- Added a conservative Faster Whisper confirmation pass that uses the currently configured wake phrase only when the unbiased transcript is already phonetically close.
|
||||
- Kept unconfirmed close transcripts in the manual review inbox instead of allowing them to become harmful negative training samples.
|
||||
- Added visible guided-transcript and review-reason details, plus retry support for ambiguous clips through Review Now.
|
||||
|
||||
9
run.sh
9
run.sh
@@ -33,8 +33,11 @@ install_ui_deps() {
|
||||
"silero-vad>=5.0.0" \
|
||||
"numpy>=1.24.0" \
|
||||
"faster-whisper>=1.0.0" \
|
||||
"onnx-asr[hub]>=0.12.0" \
|
||||
"nvidia-cublas-cu12" \
|
||||
"nvidia-cudnn-cu12==9.*"
|
||||
${PIP} uninstall -y onnxruntime
|
||||
${PIP} install "onnxruntime-gpu[cuda,cudnn]<1.27"
|
||||
}
|
||||
|
||||
# -----------------------------
|
||||
@@ -82,11 +85,13 @@ minimum = {
|
||||
"silero-vad": "5.0.0",
|
||||
"numpy": "1.24.0",
|
||||
"faster-whisper": "1.0.0",
|
||||
"onnx-asr": "0.12.0",
|
||||
"nvidia-cudnn-cu12": "9.0.0",
|
||||
}
|
||||
present = (
|
||||
"torch",
|
||||
"nvidia-cublas-cu12",
|
||||
"onnxruntime-gpu",
|
||||
)
|
||||
|
||||
for package, expected in exact.items():
|
||||
@@ -97,6 +102,10 @@ for package, minimum_version in minimum.items():
|
||||
raise SystemExit(1)
|
||||
for package in present:
|
||||
md.version(package)
|
||||
|
||||
import onnxruntime as ort
|
||||
if "CUDAExecutionProvider" not in ort.get_available_providers():
|
||||
raise SystemExit(1)
|
||||
PY
|
||||
then
|
||||
echo "UI dependencies missing or stale; installing recorder dependencies"
|
||||
|
||||
@@ -1348,7 +1348,7 @@
|
||||
</div>
|
||||
<label class="checkField">
|
||||
<input id="autoEnabled" type="checkbox" />
|
||||
<span><strong>Enable Auto Training</strong>Eligible wake triggers will be queued for local Faster Whisper transcription.</span>
|
||||
<span><strong>Enable Auto Training</strong>Eligible wake triggers will be queued for transcription with the selected local STT engine.</span>
|
||||
</label>
|
||||
<label class="checkField">
|
||||
<input id="autoDeleteConfirmedWakes" type="checkbox" />
|
||||
@@ -1368,26 +1368,10 @@
|
||||
<input id="autoLanguage" type="text" value="en" placeholder="en" />
|
||||
</label>
|
||||
<label class="field wide">
|
||||
<strong>Faster Whisper model</strong>
|
||||
<input id="autoSttModel" type="text" value="small.en" placeholder="small.en" />
|
||||
</label>
|
||||
<label class="field">
|
||||
<strong>STT device</strong>
|
||||
<select id="autoSttDevice">
|
||||
<option value="auto" selected>Auto (prefer CUDA)</option>
|
||||
<option value="cuda">CUDA</option>
|
||||
<option value="cpu">CPU</option>
|
||||
</select>
|
||||
</label>
|
||||
<label class="field">
|
||||
<strong>Compute type</strong>
|
||||
<select id="autoSttComputeType">
|
||||
<option value="auto" selected>Auto (float16 CUDA / int8 CPU)</option>
|
||||
<option value="float16">float16</option>
|
||||
<option value="int8_float16">int8_float16</option>
|
||||
<option value="int8">int8</option>
|
||||
<option value="float32">float32</option>
|
||||
<option value="default">CTranslate2 default</option>
|
||||
<strong>STT engine</strong>
|
||||
<select id="autoSttEngine">
|
||||
<option value="faster_whisper" selected>Faster Whisper (recommended)</option>
|
||||
<option value="parakeet_onnx">Parakeet ONNX</option>
|
||||
</select>
|
||||
</label>
|
||||
<label class="field">
|
||||
@@ -2091,9 +2075,7 @@
|
||||
$("autoEnabled").checked = Boolean(config.enabled);
|
||||
$("autoWakePhrase").value = config.wake_phrase || uiState.session?.raw_phrase || "";
|
||||
$("autoLanguage").value = config.language || uiState.session?.language || "en";
|
||||
$("autoSttModel").value = config.stt_model || "small.en";
|
||||
$("autoSttDevice").value = config.stt_device || "auto";
|
||||
$("autoSttComputeType").value = config.stt_compute_type || "auto";
|
||||
$("autoSttEngine").value = config.stt_engine || "faster_whisper";
|
||||
$("autoMinimumChars").value = String(config.minimum_transcript_chars ?? 2);
|
||||
$("autoDeleteConfirmedWakes").checked = Boolean(config.delete_confirmed_wakes);
|
||||
$("autoPromoteCloseMisses").checked = Boolean(config.promote_close_misses);
|
||||
@@ -2138,7 +2120,10 @@
|
||||
if (state.last_review_file) audit.push(state.last_review_file);
|
||||
if (state.last_review_transcript) audit.push(`STT: “${state.last_review_transcript}”`);
|
||||
if (state.last_review_error) audit.push(`Error: ${state.last_review_error}`);
|
||||
if (state.last_stt_device) audit.push(`STT runtime: ${state.last_stt_device} / ${state.last_stt_compute_type || "default"}`);
|
||||
if (state.last_stt_engine) {
|
||||
const runtimeLabel = [state.last_stt_device, state.last_stt_compute_type].filter(Boolean).join(" / ");
|
||||
audit.push(`STT engine: ${String(state.last_stt_engine).replaceAll("_", " ")}${runtimeLabel ? ` · ${runtimeLabel}` : ""}`);
|
||||
}
|
||||
if (state.last_notify_at) {
|
||||
audit.push(state.last_notify_error
|
||||
? `Wake-word publish failed: ${state.last_notify_error}`
|
||||
@@ -2159,9 +2144,7 @@
|
||||
enabled: $("autoEnabled").checked,
|
||||
wake_phrase: ($("autoWakePhrase").value || "").trim(),
|
||||
language: ($("autoLanguage").value || "en").trim(),
|
||||
stt_model: ($("autoSttModel").value || "").trim(),
|
||||
stt_device: $("autoSttDevice").value || "auto",
|
||||
stt_compute_type: $("autoSttComputeType").value || "auto",
|
||||
stt_engine: $("autoSttEngine").value || "faster_whisper",
|
||||
minimum_transcript_chars: Number($("autoMinimumChars").value || 2),
|
||||
delete_confirmed_wakes: $("autoDeleteConfirmedWakes").checked,
|
||||
promote_close_misses: $("autoPromoteCloseMisses").checked,
|
||||
@@ -2358,6 +2341,7 @@
|
||||
if (item.average_probability !== null && item.average_probability !== undefined) meta.push(`<span class="pill">avg ${escapeHtml(item.average_probability)}</span>`);
|
||||
if (item.detection_profile) meta.push(`<span class="pill">profile ${escapeHtml(formatDetectionProfile(item.detection_profile))}</span>`);
|
||||
if (item.auto_review_status) meta.push(`<span class="pill ${item.auto_review_status === "error" ? "err" : "warn"}">auto ${escapeHtml(String(item.auto_review_status).replaceAll("_", " "))}</span>`);
|
||||
if (item.auto_review_match_method === "guided_close_match") meta.push(`<span class="pill ok">guided wake confirmation</span>`);
|
||||
if (item.peak_probability_cutoff !== null && item.peak_probability_cutoff !== undefined) meta.push(`<span class="pill">peak cutoff ${escapeHtml(item.peak_probability_cutoff)}</span>`);
|
||||
if (item.probability_cutoff !== null && item.probability_cutoff !== undefined) meta.push(`<span class="pill">avg cutoff ${escapeHtml(item.probability_cutoff)}</span>`);
|
||||
if (item.active_window_count !== null && item.active_window_count !== undefined && item.min_active_windows !== null && item.min_active_windows !== undefined) {
|
||||
@@ -2386,6 +2370,8 @@
|
||||
</div>
|
||||
<div class="fileMeta">${meta.join("") || `<span class="muted">No metadata attached</span>`}</div>
|
||||
${item.transcript ? `<div class="autoAudit"><strong>STT transcript</strong><br>${escapeHtml(item.transcript)}</div>` : ""}
|
||||
${item.auto_review_guided_transcript ? `<div class="autoAudit"><strong>Guided wake check</strong><br>${escapeHtml(item.auto_review_guided_transcript)}</div>` : ""}
|
||||
${item.auto_review_reason ? `<div class="muted">${escapeHtml(item.auto_review_reason)}</div>` : ""}
|
||||
${item.auto_review_error ? `<div class="muted">Auto review error: ${escapeHtml(item.auto_review_error)}</div>` : ""}
|
||||
<audio class="audioPlayer" controls preload="none" src="${escapeHtml(item.audio_url || `/api/audio/captured/${encodeURIComponent(item.saved_as)}`)}"></audio>
|
||||
<div class="muted">Stored as ${escapeHtml(item.saved_as)} · ${escapeHtml(formatSummary)}</div>
|
||||
|
||||
@@ -43,12 +43,14 @@ class AutoTrainTests(unittest.TestCase):
|
||||
trainer.PERSONAL_DIR,
|
||||
trainer.AUTO_TRAIN_CONFIG_FILE,
|
||||
trainer.AUTO_TRAIN_STATE_FILE,
|
||||
trainer.AUTO_TRAIN_MODEL_DIR,
|
||||
)
|
||||
trainer.CAPTURED_DIR = root / "captured_audio"
|
||||
trainer.NEGATIVE_DIR = root / "negative_samples"
|
||||
trainer.PERSONAL_DIR = root / "personal_samples"
|
||||
trainer.AUTO_TRAIN_CONFIG_FILE = root / "auto_train_config.json"
|
||||
trainer.AUTO_TRAIN_STATE_FILE = root / "auto_train_state.json"
|
||||
trainer.AUTO_TRAIN_MODEL_DIR = root / "auto_train_models"
|
||||
for directory in (trainer.CAPTURED_DIR, trainer.NEGATIVE_DIR, trainer.PERSONAL_DIR):
|
||||
directory.mkdir(parents=True)
|
||||
|
||||
@@ -62,8 +64,6 @@ class AutoTrainTests(unittest.TestCase):
|
||||
"wake_phrase": "hey tater",
|
||||
"language": "en",
|
||||
"tater_url": "http://127.0.0.1:8501",
|
||||
"stt_device": "auto",
|
||||
"stt_compute_type": "auto",
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -77,6 +77,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
trainer.PERSONAL_DIR,
|
||||
trainer.AUTO_TRAIN_CONFIG_FILE,
|
||||
trainer.AUTO_TRAIN_STATE_FILE,
|
||||
trainer.AUTO_TRAIN_MODEL_DIR,
|
||||
) = self.original_paths
|
||||
trainer.AUTO_TRAIN_CONFIG.clear()
|
||||
trainer.AUTO_TRAIN_CONFIG.update(self.original_config)
|
||||
@@ -110,9 +111,199 @@ class AutoTrainTests(unittest.TestCase):
|
||||
self.assertTrue(trainer._transcript_contains_wake_phrase("Okay, HEY TATER!", "hey_tater"))
|
||||
self.assertFalse(trainer._transcript_contains_wake_phrase("Turn on the television", "hey tater"))
|
||||
|
||||
def test_phrase_similarity_recognizes_real_short_clip_mishearings(self):
|
||||
for transcript in ("Hey, haters.", "Hate hater.", "Hey Ganger.", "Hey, gator."):
|
||||
with self.subTest(transcript=transcript):
|
||||
self.assertGreaterEqual(
|
||||
trainer._wake_phrase_similarity(transcript, "hey tater"),
|
||||
trainer.WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY,
|
||||
)
|
||||
|
||||
for transcript in ("turn on the lights", "what is the weather", "play some music"):
|
||||
with self.subTest(transcript=transcript):
|
||||
self.assertLess(
|
||||
trainer._wake_phrase_similarity(transcript, "hey tater"),
|
||||
trainer.WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY,
|
||||
)
|
||||
|
||||
def test_stt_engine_selection_uses_managed_models(self):
|
||||
config = trainer._normalize_auto_train_config(
|
||||
{
|
||||
"stt_engine": "parakeet-onnx",
|
||||
"stt_model": "user/should-not-be-used",
|
||||
"stt_device": "cpu",
|
||||
"stt_compute_type": "float32",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(config["stt_engine"], trainer.STT_ENGINE_PARAKEET_ONNX)
|
||||
self.assertNotIn("stt_model", config)
|
||||
self.assertNotIn("stt_device", config)
|
||||
self.assertNotIn("stt_compute_type", config)
|
||||
self.assertEqual(
|
||||
trainer._managed_stt_model(config["stt_engine"], "en"),
|
||||
trainer.DEFAULT_PARAKEET_ONNX_MODEL,
|
||||
)
|
||||
self.assertEqual(
|
||||
trainer._managed_stt_model(trainer.STT_ENGINE_FASTER_WHISPER, "de"),
|
||||
trainer.DEFAULT_FASTER_WHISPER_MULTILINGUAL_MODEL,
|
||||
)
|
||||
|
||||
def test_stt_router_supports_both_nvidia_engines(self):
|
||||
audio_path = Path("wake.wav")
|
||||
with (
|
||||
patch.object(trainer, "_transcribe_capture_with_faster_whisper", return_value="faster") as faster,
|
||||
patch.object(trainer, "_transcribe_capture_with_parakeet", return_value="parakeet") as parakeet,
|
||||
):
|
||||
self.assertEqual(
|
||||
trainer._transcribe_capture(
|
||||
audio_path,
|
||||
engine=trainer.STT_ENGINE_FASTER_WHISPER,
|
||||
language="en",
|
||||
),
|
||||
"faster",
|
||||
)
|
||||
self.assertEqual(
|
||||
trainer._transcribe_capture(
|
||||
audio_path,
|
||||
engine=trainer.STT_ENGINE_PARAKEET_ONNX,
|
||||
language="en",
|
||||
),
|
||||
"parakeet",
|
||||
)
|
||||
|
||||
faster.assert_called_once()
|
||||
parakeet.assert_called_once()
|
||||
|
||||
def test_guided_faster_whisper_uses_dynamic_wake_phrase(self):
|
||||
fake_model = SimpleNamespace(
|
||||
transcribe=Mock(
|
||||
return_value=(
|
||||
iter([SimpleNamespace(text=" hello "), SimpleNamespace(text="potato ")]),
|
||||
SimpleNamespace(),
|
||||
)
|
||||
)
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
trainer,
|
||||
"_resolve_faster_whisper_runtime",
|
||||
return_value=("cuda", "float16"),
|
||||
),
|
||||
patch.object(trainer, "_load_faster_whisper_model", return_value=fake_model),
|
||||
):
|
||||
transcript = trainer._transcribe_capture_with_faster_whisper_guided(
|
||||
Path("wake.wav"),
|
||||
model="small.en",
|
||||
language="en",
|
||||
wake_phrase="Hello_Potato",
|
||||
)
|
||||
|
||||
self.assertEqual(transcript, "hello potato")
|
||||
_, kwargs = fake_model.transcribe.call_args
|
||||
self.assertEqual(kwargs["hotwords"], "hello potato")
|
||||
self.assertIn("hello potato", kwargs["initial_prompt"])
|
||||
self.assertEqual(kwargs["beam_size"], 5)
|
||||
self.assertEqual(kwargs["best_of"], 5)
|
||||
self.assertEqual(kwargs["temperature"], 0.0)
|
||||
self.assertFalse(kwargs["condition_on_previous_text"])
|
||||
|
||||
def test_parakeet_loader_prefers_cuda_then_cpu(self):
|
||||
fake_model = object()
|
||||
fake_onnx_asr = SimpleNamespace(load_model=Mock(return_value=fake_model))
|
||||
fake_huggingface_hub = SimpleNamespace(
|
||||
snapshot_download=Mock(return_value=str(trainer.AUTO_TRAIN_MODEL_DIR))
|
||||
)
|
||||
with (
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"onnx_asr": fake_onnx_asr,
|
||||
"huggingface_hub": fake_huggingface_hub,
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
trainer,
|
||||
"_parakeet_onnx_providers",
|
||||
return_value=["CUDAExecutionProvider", "CPUExecutionProvider"],
|
||||
),
|
||||
):
|
||||
with trainer.PARAKEET_ONNX_MODEL_LOCK:
|
||||
trainer.PARAKEET_ONNX_MODEL_CACHE.clear()
|
||||
loaded = trainer._load_parakeet_onnx_model()
|
||||
|
||||
self.assertIs(loaded, fake_model)
|
||||
fake_huggingface_hub.snapshot_download.assert_called_once_with(
|
||||
repo_id=trainer.DEFAULT_PARAKEET_ONNX_REPO,
|
||||
local_dir=str(trainer.AUTO_TRAIN_MODEL_DIR),
|
||||
allow_patterns=[
|
||||
"config.json",
|
||||
"vocab.txt",
|
||||
"encoder-model.int8.onnx",
|
||||
"encoder-model.int8.onnx.data",
|
||||
"decoder_joint-model.int8.onnx",
|
||||
"decoder_joint-model.int8.onnx.data",
|
||||
],
|
||||
)
|
||||
fake_onnx_asr.load_model.assert_called_once_with(
|
||||
trainer.DEFAULT_PARAKEET_ONNX_MODEL,
|
||||
str(trainer.AUTO_TRAIN_MODEL_DIR),
|
||||
quantization="int8",
|
||||
providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
|
||||
)
|
||||
|
||||
def test_parakeet_loader_reuses_complete_snapshot_offline(self):
|
||||
fake_model = object()
|
||||
fake_onnx_asr = SimpleNamespace(load_model=Mock(return_value=fake_model))
|
||||
fake_huggingface_hub = SimpleNamespace(snapshot_download=Mock())
|
||||
trainer.AUTO_TRAIN_MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
for filename in (
|
||||
"config.json",
|
||||
"vocab.txt",
|
||||
"encoder-model.int8.onnx",
|
||||
"decoder_joint-model.int8.onnx",
|
||||
):
|
||||
(trainer.AUTO_TRAIN_MODEL_DIR / filename).touch()
|
||||
with (
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"onnx_asr": fake_onnx_asr,
|
||||
"huggingface_hub": fake_huggingface_hub,
|
||||
},
|
||||
),
|
||||
patch.object(
|
||||
trainer,
|
||||
"_parakeet_onnx_providers",
|
||||
return_value=["CUDAExecutionProvider", "CPUExecutionProvider"],
|
||||
),
|
||||
):
|
||||
with trainer.PARAKEET_ONNX_MODEL_LOCK:
|
||||
trainer.PARAKEET_ONNX_MODEL_CACHE.clear()
|
||||
loaded = trainer._load_parakeet_onnx_model()
|
||||
|
||||
self.assertIs(loaded, fake_model)
|
||||
fake_huggingface_hub.snapshot_download.assert_not_called()
|
||||
fake_onnx_asr.load_model.assert_called_once_with(
|
||||
trainer.DEFAULT_PARAKEET_ONNX_MODEL,
|
||||
str(trainer.AUTO_TRAIN_MODEL_DIR),
|
||||
quantization="int8",
|
||||
providers=["CUDAExecutionProvider", "CPUExecutionProvider"],
|
||||
)
|
||||
|
||||
def test_ui_exposes_engine_selector_without_manual_runtime_fields(self):
|
||||
source = (Path(__file__).resolve().parents[1] / "static" / "index.html").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
self.assertIn('id="autoSttEngine"', source)
|
||||
self.assertNotIn('id="autoSttModel"', source)
|
||||
self.assertNotIn('id="autoSttDevice"', source)
|
||||
self.assertNotIn('id="autoSttComputeType"', source)
|
||||
self.assertIn("Guided wake check", source)
|
||||
|
||||
def test_phrase_miss_moves_wake_trigger_to_negative_samples(self):
|
||||
self.add_capture()
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper", return_value="turn on the kitchen lights"):
|
||||
with patch.object(trainer, "_transcribe_capture", return_value="turn on the kitchen lights"):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
self.assertFalse((trainer.CAPTURED_DIR / "wake.wav").exists())
|
||||
@@ -122,11 +313,13 @@ class AutoTrainTests(unittest.TestCase):
|
||||
self.assertTrue(metadata["auto_negative"])
|
||||
self.assertEqual(metadata["review_status"], "auto_approved_negative")
|
||||
self.assertEqual(metadata["transcript"], "turn on the kitchen lights")
|
||||
self.assertEqual(metadata["auto_review_stt_engine"], "faster_whisper")
|
||||
self.assertEqual(metadata["auto_review_stt_model"], "small.en")
|
||||
self.assertEqual(trainer.AUTO_TRAIN_STATE["pending_negative_count"], 1)
|
||||
|
||||
def test_matching_phrase_stays_in_manual_review_inbox(self):
|
||||
audio_path = self.add_capture()
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper", return_value="hey tater turn on the lights"):
|
||||
with patch.object(trainer, "_transcribe_capture", return_value="hey tater turn on the lights"):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
self.assertTrue(audio_path.exists())
|
||||
@@ -135,12 +328,86 @@ class AutoTrainTests(unittest.TestCase):
|
||||
self.assertEqual(metadata["auto_review_status"], "wake_phrase_detected")
|
||||
self.assertEqual(trainer.AUTO_TRAIN_STATE["pending_negative_count"], 0)
|
||||
|
||||
def test_close_transcript_uses_guided_faster_whisper_confirmation(self):
|
||||
audio_path = self.add_capture()
|
||||
with (
|
||||
patch.object(trainer, "_transcribe_capture", return_value="Hey, haters."),
|
||||
patch.object(
|
||||
trainer,
|
||||
"_transcribe_capture_with_faster_whisper_guided",
|
||||
return_value="Hey Tater",
|
||||
) as guided,
|
||||
):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
self.assertTrue(audio_path.exists())
|
||||
self.assertFalse(list(trainer.NEGATIVE_DIR.glob("*.wav")))
|
||||
metadata = trainer._load_sidecar_json(audio_path)
|
||||
self.assertEqual(metadata["auto_review_status"], "wake_phrase_detected")
|
||||
self.assertEqual(metadata["transcript"], "Hey, haters.")
|
||||
self.assertEqual(metadata["auto_review_guided_transcript"], "Hey Tater")
|
||||
self.assertEqual(metadata["auto_review_match_method"], "guided_close_match")
|
||||
self.assertGreaterEqual(
|
||||
metadata["auto_review_phrase_similarity"],
|
||||
trainer.WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY,
|
||||
)
|
||||
guided.assert_called_once()
|
||||
guided_args, guided_kwargs = guided.call_args
|
||||
self.assertEqual(guided_args[0].resolve(), audio_path.resolve())
|
||||
self.assertEqual(
|
||||
guided_kwargs,
|
||||
{
|
||||
"model": "small.en",
|
||||
"language": "en",
|
||||
"wake_phrase": "hey tater",
|
||||
},
|
||||
)
|
||||
|
||||
def test_unconfirmed_close_transcript_stays_for_manual_review(self):
|
||||
audio_path = self.add_capture()
|
||||
with (
|
||||
patch.object(trainer, "_transcribe_capture", return_value="Hate hater."),
|
||||
patch.object(
|
||||
trainer,
|
||||
"_transcribe_capture_with_faster_whisper_guided",
|
||||
return_value="Hate hater.",
|
||||
),
|
||||
):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
self.assertTrue(audio_path.exists())
|
||||
self.assertFalse(list(trainer.NEGATIVE_DIR.glob("*.wav")))
|
||||
metadata = trainer._load_sidecar_json(audio_path)
|
||||
self.assertEqual(metadata["auto_review_status"], "wake_phrase_ambiguous")
|
||||
self.assertEqual(metadata["transcript"], "Hate hater.")
|
||||
self.assertEqual(metadata["auto_review_guided_transcript"], "Hate hater.")
|
||||
self.assertEqual(trainer.AUTO_TRAIN_STATE["pending_negative_count"], 0)
|
||||
|
||||
self.assertEqual(trainer._queue_pending_auto_reviews(), 0)
|
||||
self.assertEqual(trainer._queue_pending_auto_reviews(force=True), 1)
|
||||
|
||||
def test_close_parakeet_transcript_stays_for_manual_review(self):
|
||||
audio_path = self.add_capture()
|
||||
trainer.AUTO_TRAIN_CONFIG["stt_engine"] = trainer.STT_ENGINE_PARAKEET_ONNX
|
||||
with (
|
||||
patch.object(trainer, "_transcribe_capture", return_value="Hey Ganger."),
|
||||
patch.object(trainer, "_transcribe_capture_with_faster_whisper_guided") as guided,
|
||||
):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
guided.assert_not_called()
|
||||
self.assertTrue(audio_path.exists())
|
||||
self.assertFalse(list(trainer.NEGATIVE_DIR.glob("*.wav")))
|
||||
metadata = trainer._load_sidecar_json(audio_path)
|
||||
self.assertEqual(metadata["auto_review_status"], "wake_phrase_ambiguous")
|
||||
self.assertEqual(metadata["auto_review_stt_engine"], "parakeet_onnx")
|
||||
|
||||
def test_matching_phrase_is_deleted_when_cleanup_is_enabled(self):
|
||||
audio_path = self.add_capture()
|
||||
trainer.AUTO_TRAIN_CONFIG["delete_confirmed_wakes"] = True
|
||||
with patch.object(
|
||||
trainer,
|
||||
"_transcribe_capture_with_faster_whisper",
|
||||
"_transcribe_capture",
|
||||
return_value="hey tater turn on the lights",
|
||||
):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
@@ -164,7 +431,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
trainer.AUTO_TRAIN_CONFIG["delete_confirmed_wakes"] = True
|
||||
|
||||
self.assertEqual(trainer._queue_pending_auto_reviews(), 1)
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper") as transcribe:
|
||||
with patch.object(trainer, "_transcribe_capture") as transcribe:
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
transcribe.assert_not_called()
|
||||
@@ -173,7 +440,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
|
||||
def test_close_miss_is_not_transcribed_by_default(self):
|
||||
audio_path = self.add_capture(event_type="close_miss")
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper") as transcribe:
|
||||
with patch.object(trainer, "_transcribe_capture") as transcribe:
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
transcribe.assert_not_called()
|
||||
@@ -190,7 +457,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
def test_close_miss_with_phrase_is_promoted_when_enabled(self):
|
||||
self.add_capture(event_type="close_miss")
|
||||
trainer.AUTO_TRAIN_CONFIG["promote_close_misses"] = True
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper", return_value="hey tater"):
|
||||
with patch.object(trainer, "_transcribe_capture", return_value="hey tater"):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
self.assertFalse((trainer.CAPTURED_DIR / "wake.wav").exists())
|
||||
@@ -208,7 +475,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
trainer.AUTO_TRAIN_CONFIG["promote_close_misses"] = True
|
||||
with patch.object(
|
||||
trainer,
|
||||
"_transcribe_capture_with_faster_whisper",
|
||||
"_transcribe_capture",
|
||||
return_value="turn on the lights",
|
||||
):
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
@@ -222,7 +489,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
def test_vad_blocked_close_miss_is_never_transcribed(self):
|
||||
audio_path = self.add_capture(event_type="close_miss", blocked_by_vad=True)
|
||||
trainer.AUTO_TRAIN_CONFIG["promote_close_misses"] = True
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper") as transcribe:
|
||||
with patch.object(trainer, "_transcribe_capture") as transcribe:
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
transcribe.assert_not_called()
|
||||
@@ -231,7 +498,7 @@ class AutoTrainTests(unittest.TestCase):
|
||||
|
||||
def test_capture_for_another_wake_word_is_not_transcribed(self):
|
||||
audio_path = self.add_capture(wake_word="computer")
|
||||
with patch.object(trainer, "_transcribe_capture_with_faster_whisper") as transcribe:
|
||||
with patch.object(trainer, "_transcribe_capture") as transcribe:
|
||||
trainer._auto_review_capture("wake.wav")
|
||||
|
||||
transcribe.assert_not_called()
|
||||
|
||||
@@ -61,6 +61,13 @@ class RunShCudaLibraryPathTests(unittest.TestCase):
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), "")
|
||||
|
||||
def test_parakeet_uses_cuda_onnxruntime_package(self) -> None:
|
||||
source = RUN_SH.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn('"onnx-asr[hub]>=0.12.0"', source)
|
||||
self.assertIn('"onnxruntime-gpu[cuda,cudnn]<1.27"', source)
|
||||
self.assertIn('"CUDAExecutionProvider"', source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
# trainer_server.py
|
||||
import contextlib
|
||||
import gc
|
||||
import io
|
||||
import os
|
||||
import queue
|
||||
@@ -19,6 +20,7 @@ import unicodedata
|
||||
import wave
|
||||
from array import array
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from difflib import SequenceMatcher
|
||||
from math import isfinite, log10
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Callable, Optional, Tuple
|
||||
@@ -88,15 +90,42 @@ TARGET_SAMPLE_RATE = 16000
|
||||
TARGET_CHANNELS = 1
|
||||
TARGET_SAMPLE_WIDTH_BYTES = 2
|
||||
CAPTURE_GAIN_PROFILE = "capture_rms_v1"
|
||||
DEFAULT_FASTER_WHISPER_MODEL = os.environ.get("AUTO_TRAIN_STT_MODEL", "small.en")
|
||||
STT_ENGINE_FASTER_WHISPER = "faster_whisper"
|
||||
STT_ENGINE_PARAKEET_ONNX = "parakeet_onnx"
|
||||
SUPPORTED_STT_ENGINES = {
|
||||
STT_ENGINE_FASTER_WHISPER,
|
||||
STT_ENGINE_PARAKEET_ONNX,
|
||||
}
|
||||
DEFAULT_STT_ENGINE = os.environ.get(
|
||||
"AUTO_TRAIN_STT_ENGINE",
|
||||
STT_ENGINE_FASTER_WHISPER,
|
||||
).strip().lower().replace("-", "_")
|
||||
if DEFAULT_STT_ENGINE not in SUPPORTED_STT_ENGINES:
|
||||
DEFAULT_STT_ENGINE = STT_ENGINE_FASTER_WHISPER
|
||||
DEFAULT_FASTER_WHISPER_EN_MODEL = os.environ.get(
|
||||
"AUTO_TRAIN_FASTER_WHISPER_EN_MODEL",
|
||||
"small.en",
|
||||
)
|
||||
DEFAULT_FASTER_WHISPER_MULTILINGUAL_MODEL = os.environ.get(
|
||||
"AUTO_TRAIN_FASTER_WHISPER_MULTILINGUAL_MODEL",
|
||||
"small",
|
||||
)
|
||||
DEFAULT_PARAKEET_ONNX_MODEL = os.environ.get(
|
||||
"AUTO_TRAIN_PARAKEET_ONNX_MODEL",
|
||||
"nemo-parakeet-tdt-0.6b-v3",
|
||||
)
|
||||
DEFAULT_PARAKEET_ONNX_REPO = os.environ.get(
|
||||
"AUTO_TRAIN_PARAKEET_ONNX_REPO",
|
||||
"istupakov/parakeet-tdt-0.6b-v3-onnx",
|
||||
)
|
||||
DEFAULT_PARAKEET_ONNX_QUANTIZATION = "int8"
|
||||
WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY = 0.68
|
||||
|
||||
AUTO_TRAIN_DEFAULT_CONFIG: Dict[str, Any] = {
|
||||
"enabled": False,
|
||||
"wake_phrase": "",
|
||||
"language": DEFAULT_LANGUAGE,
|
||||
"stt_model": DEFAULT_FASTER_WHISPER_MODEL,
|
||||
"stt_device": "auto",
|
||||
"stt_compute_type": "auto",
|
||||
"stt_engine": DEFAULT_STT_ENGINE,
|
||||
"minimum_transcript_chars": 2,
|
||||
"delete_confirmed_wakes": False,
|
||||
"promote_close_misses": False,
|
||||
@@ -119,6 +148,8 @@ AUTO_TRAIN_DEFAULT_STATE: Dict[str, Any] = {
|
||||
"last_review_transcript": "",
|
||||
"last_review_result": "",
|
||||
"last_review_error": "",
|
||||
"last_stt_engine": "",
|
||||
"last_stt_model": "",
|
||||
"last_stt_device": "",
|
||||
"last_stt_compute_type": "",
|
||||
"last_train_started_at": "",
|
||||
@@ -186,6 +217,10 @@ AUTO_TRAIN_RUNTIME: Dict[str, Any] = {
|
||||
LAN_ADDRESS_CACHE: Dict[str, Any] = {"value": "", "fetched_at": 0.0}
|
||||
FASTER_WHISPER_MODEL_LOCK = threading.RLock()
|
||||
FASTER_WHISPER_MODEL_CACHE: Dict[Tuple[str, str, str], Any] = {}
|
||||
FASTER_WHISPER_TRANSCRIBE_LOCK = threading.RLock()
|
||||
PARAKEET_ONNX_MODEL_LOCK = threading.RLock()
|
||||
PARAKEET_ONNX_MODEL_CACHE: Dict[Tuple[str, str, Tuple[str, ...]], Any] = {}
|
||||
PARAKEET_ONNX_TRANSCRIBE_LOCK = threading.RLock()
|
||||
PIPER_CATALOG_CACHE: Dict[str, Any] = {
|
||||
"fetched_at": 0.0,
|
||||
"entries": None,
|
||||
@@ -461,25 +496,60 @@ def _normalize_http_base_url(value: Any, *, allow_empty: bool = True) -> str:
|
||||
return token
|
||||
|
||||
|
||||
def _normalize_stt_engine(value: Any) -> str:
|
||||
token = str(value or DEFAULT_STT_ENGINE).strip().lower().replace("-", "_")
|
||||
aliases = {
|
||||
"faster": STT_ENGINE_FASTER_WHISPER,
|
||||
"fasterwhisper": STT_ENGINE_FASTER_WHISPER,
|
||||
"parakeet": STT_ENGINE_PARAKEET_ONNX,
|
||||
"onnx_parakeet": STT_ENGINE_PARAKEET_ONNX,
|
||||
}
|
||||
token = aliases.get(token, token)
|
||||
if token not in SUPPORTED_STT_ENGINES:
|
||||
raise ValueError("STT engine must be Faster Whisper or Parakeet ONNX.")
|
||||
return token
|
||||
|
||||
|
||||
def _managed_stt_model(engine: Any, language: Any = DEFAULT_LANGUAGE) -> str:
|
||||
token = _normalize_stt_engine(engine)
|
||||
language_token = str(language or DEFAULT_LANGUAGE).strip().lower().replace("-", "_")
|
||||
english = language_token == "en" or language_token.startswith("en_")
|
||||
if token == STT_ENGINE_FASTER_WHISPER:
|
||||
return (
|
||||
DEFAULT_FASTER_WHISPER_EN_MODEL
|
||||
if english
|
||||
else DEFAULT_FASTER_WHISPER_MULTILINGUAL_MODEL
|
||||
)
|
||||
return DEFAULT_PARAKEET_ONNX_MODEL
|
||||
|
||||
|
||||
def _stt_engine_catalog(language: Any = DEFAULT_LANGUAGE) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"value": STT_ENGINE_FASTER_WHISPER,
|
||||
"label": "Faster Whisper",
|
||||
"model": _managed_stt_model(STT_ENGINE_FASTER_WHISPER, language),
|
||||
"recommended": True,
|
||||
},
|
||||
{
|
||||
"value": STT_ENGINE_PARAKEET_ONNX,
|
||||
"label": "Parakeet ONNX",
|
||||
"model": _managed_stt_model(STT_ENGINE_PARAKEET_ONNX, language),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _normalize_auto_train_config(values: Dict[str, Any] | None, *, base: Dict[str, Any] | None = None) -> Dict[str, Any]:
|
||||
incoming = values if isinstance(values, dict) else {}
|
||||
source = {**AUTO_TRAIN_DEFAULT_CONFIG, **(base or {}), **incoming}
|
||||
schedule_hours = _bounded_int(source.get("schedule_hours"), 24, 0, 24 * 30)
|
||||
language = str(source.get("language") or DEFAULT_LANGUAGE).strip().lower().replace("-", "_")
|
||||
language = re.sub(r"[^a-z0-9_]", "", language) or DEFAULT_LANGUAGE
|
||||
stt_device = str(source.get("stt_device") or "auto").strip().lower()
|
||||
if stt_device not in {"auto", "cuda", "cpu"}:
|
||||
raise ValueError("Faster Whisper device must be auto, cuda, or cpu.")
|
||||
stt_compute_type = str(source.get("stt_compute_type") or "auto").strip().lower()
|
||||
if stt_compute_type not in {"auto", "default", "float16", "float32", "int8", "int8_float16"}:
|
||||
raise ValueError("Unsupported Faster Whisper compute type.")
|
||||
return {
|
||||
"enabled": _config_bool(source.get("enabled")),
|
||||
"wake_phrase": str(source.get("wake_phrase") or "").strip(),
|
||||
"language": language,
|
||||
"stt_model": str(source.get("stt_model") or DEFAULT_FASTER_WHISPER_MODEL).strip() or DEFAULT_FASTER_WHISPER_MODEL,
|
||||
"stt_device": stt_device,
|
||||
"stt_compute_type": stt_compute_type,
|
||||
"stt_engine": _normalize_stt_engine(source.get("stt_engine")),
|
||||
"minimum_transcript_chars": _bounded_int(source.get("minimum_transcript_chars"), 2, 1, 100),
|
||||
"delete_confirmed_wakes": _config_bool(source.get("delete_confirmed_wakes")),
|
||||
"promote_close_misses": _config_bool(source.get("promote_close_misses")),
|
||||
@@ -555,10 +625,12 @@ def _public_auto_train_config() -> Dict[str, Any]:
|
||||
|
||||
def _auto_train_status_payload() -> Dict[str, Any]:
|
||||
with AUTO_TRAIN_LOCK:
|
||||
language = AUTO_TRAIN_CONFIG.get("language") or DEFAULT_LANGUAGE
|
||||
return {
|
||||
"config": _public_auto_train_config(),
|
||||
"state": dict(AUTO_TRAIN_STATE),
|
||||
"runtime": dict(AUTO_TRAIN_RUNTIME),
|
||||
"stt_engines": _stt_engine_catalog(language),
|
||||
"advertised_base_url": _advertised_base_url(),
|
||||
"trainer_link": _tater_link_public_status(),
|
||||
}
|
||||
@@ -765,6 +837,28 @@ def _transcript_contains_wake_phrase(transcript: Any, wake_phrase: Any) -> bool:
|
||||
return f" {normalized_phrase} " in f" {normalized_transcript} "
|
||||
|
||||
|
||||
def _wake_phrase_similarity(transcript: Any, wake_phrase: Any) -> float:
|
||||
transcript_words = _normalize_transcript_text(transcript).split()
|
||||
phrase_words = _normalize_transcript_text(wake_phrase).split()
|
||||
if not transcript_words or not phrase_words:
|
||||
return 0.0
|
||||
if _transcript_contains_wake_phrase(transcript, wake_phrase):
|
||||
return 1.0
|
||||
|
||||
phrase_token = "".join(phrase_words)
|
||||
minimum_words = max(1, len(phrase_words) - 1)
|
||||
maximum_words = min(len(transcript_words), len(phrase_words) + 1)
|
||||
best_score = 0.0
|
||||
for word_count in range(minimum_words, maximum_words + 1):
|
||||
for start in range(0, len(transcript_words) - word_count + 1):
|
||||
candidate = "".join(transcript_words[start : start + word_count])
|
||||
best_score = max(
|
||||
best_score,
|
||||
SequenceMatcher(None, candidate, phrase_token).ratio(),
|
||||
)
|
||||
return best_score
|
||||
|
||||
|
||||
def _captured_event_is_close_miss(metadata: Dict[str, Any]) -> bool:
|
||||
event_type = str(metadata.get("event_type") or "captured").strip().lower()
|
||||
return "close" in event_type
|
||||
@@ -828,33 +922,232 @@ def _load_faster_whisper_model(*, model_name: str, device: str, compute_type: st
|
||||
|
||||
|
||||
def _transcribe_capture_with_faster_whisper(audio_path: Path, *, model: str, language: str) -> str:
|
||||
with AUTO_TRAIN_LOCK:
|
||||
device_value = AUTO_TRAIN_CONFIG.get("stt_device")
|
||||
compute_value = AUTO_TRAIN_CONFIG.get("stt_compute_type")
|
||||
device, compute_type = _resolve_faster_whisper_runtime(device_value, compute_value)
|
||||
device, compute_type = _resolve_faster_whisper_runtime("auto", "auto")
|
||||
whisper_model = _load_faster_whisper_model(
|
||||
model_name=model,
|
||||
device=device,
|
||||
compute_type=compute_type,
|
||||
)
|
||||
segments, _info = whisper_model.transcribe(
|
||||
str(audio_path),
|
||||
language=language or None,
|
||||
beam_size=1,
|
||||
condition_on_previous_text=False,
|
||||
)
|
||||
transcript = re.sub(
|
||||
r"\s+",
|
||||
" ",
|
||||
" ".join(str(segment.text or "").strip() for segment in segments),
|
||||
).strip()
|
||||
with FASTER_WHISPER_TRANSCRIBE_LOCK:
|
||||
segments, _info = whisper_model.transcribe(
|
||||
str(audio_path),
|
||||
language=language or None,
|
||||
beam_size=1,
|
||||
condition_on_previous_text=False,
|
||||
)
|
||||
transcript = re.sub(
|
||||
r"\s+",
|
||||
" ",
|
||||
" ".join(str(segment.text or "").strip() for segment in segments),
|
||||
).strip()
|
||||
with AUTO_TRAIN_LOCK:
|
||||
AUTO_TRAIN_STATE["last_stt_engine"] = STT_ENGINE_FASTER_WHISPER
|
||||
AUTO_TRAIN_STATE["last_stt_model"] = model
|
||||
AUTO_TRAIN_STATE["last_stt_device"] = device
|
||||
AUTO_TRAIN_STATE["last_stt_compute_type"] = compute_type
|
||||
_save_auto_train_state_locked()
|
||||
return transcript
|
||||
|
||||
|
||||
def _transcribe_capture_with_faster_whisper_guided(
|
||||
audio_path: Path,
|
||||
*,
|
||||
model: str,
|
||||
language: str,
|
||||
wake_phrase: str,
|
||||
) -> str:
|
||||
normalized_phrase = _normalize_transcript_text(wake_phrase)
|
||||
if not normalized_phrase:
|
||||
return ""
|
||||
device, compute_type = _resolve_faster_whisper_runtime("auto", "auto")
|
||||
whisper_model = _load_faster_whisper_model(
|
||||
model_name=model,
|
||||
device=device,
|
||||
compute_type=compute_type,
|
||||
)
|
||||
with FASTER_WHISPER_TRANSCRIBE_LOCK:
|
||||
segments, _info = whisper_model.transcribe(
|
||||
str(audio_path),
|
||||
language=language or None,
|
||||
beam_size=5,
|
||||
best_of=5,
|
||||
temperature=0.0,
|
||||
condition_on_previous_text=False,
|
||||
initial_prompt=f'The wake phrase is "{normalized_phrase}".',
|
||||
hotwords=normalized_phrase,
|
||||
)
|
||||
return re.sub(
|
||||
r"\s+",
|
||||
" ",
|
||||
" ".join(str(segment.text or "").strip() for segment in segments),
|
||||
).strip()
|
||||
|
||||
|
||||
def _parakeet_onnx_providers() -> List[str]:
|
||||
try:
|
||||
import onnxruntime as ort
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"onnxruntime is unavailable: {exc}") from exc
|
||||
available = [str(value) for value in ort.get_available_providers()]
|
||||
preferred = [
|
||||
"CUDAExecutionProvider",
|
||||
"CPUExecutionProvider",
|
||||
]
|
||||
resolved = [provider for provider in preferred if provider in set(available)]
|
||||
if not resolved:
|
||||
raise RuntimeError("ONNX Runtime has no usable CUDA or CPU execution provider.")
|
||||
return resolved
|
||||
|
||||
|
||||
def _load_parakeet_onnx_model():
|
||||
try:
|
||||
import onnx_asr
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"onnx-asr is unavailable: {exc}") from exc
|
||||
|
||||
providers = tuple(_parakeet_onnx_providers())
|
||||
cache_key = (
|
||||
DEFAULT_PARAKEET_ONNX_MODEL,
|
||||
DEFAULT_PARAKEET_ONNX_QUANTIZATION,
|
||||
providers,
|
||||
)
|
||||
with PARAKEET_ONNX_MODEL_LOCK:
|
||||
cached = PARAKEET_ONNX_MODEL_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
suffix = (
|
||||
f".{DEFAULT_PARAKEET_ONNX_QUANTIZATION}"
|
||||
if DEFAULT_PARAKEET_ONNX_QUANTIZATION
|
||||
else ""
|
||||
)
|
||||
model_patterns = [
|
||||
"config.json",
|
||||
"vocab.txt",
|
||||
f"encoder-model{suffix}.onnx",
|
||||
f"encoder-model{suffix}.onnx.data",
|
||||
f"decoder_joint-model{suffix}.onnx",
|
||||
f"decoder_joint-model{suffix}.onnx.data",
|
||||
]
|
||||
required_model_files = [
|
||||
"config.json",
|
||||
"vocab.txt",
|
||||
f"encoder-model{suffix}.onnx",
|
||||
f"decoder_joint-model{suffix}.onnx",
|
||||
]
|
||||
if not DEFAULT_PARAKEET_ONNX_QUANTIZATION:
|
||||
required_model_files.append("encoder-model.onnx.data")
|
||||
AUTO_TRAIN_MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
previous = {
|
||||
key: os.environ.get(key)
|
||||
for key in ("HF_HOME", "HF_HUB_CACHE", "HUGGINGFACE_HUB_CACHE")
|
||||
}
|
||||
os.environ["HF_HOME"] = str(AUTO_TRAIN_MODEL_DIR)
|
||||
os.environ["HF_HUB_CACHE"] = str(AUTO_TRAIN_MODEL_DIR / "hub")
|
||||
os.environ["HUGGINGFACE_HUB_CACHE"] = str(AUTO_TRAIN_MODEL_DIR / "hub")
|
||||
try:
|
||||
snapshot_root = AUTO_TRAIN_MODEL_DIR
|
||||
if not all(
|
||||
(AUTO_TRAIN_MODEL_DIR / filename).is_file()
|
||||
for filename in required_model_files
|
||||
):
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
snapshot_root = Path(
|
||||
snapshot_download(
|
||||
repo_id=DEFAULT_PARAKEET_ONNX_REPO,
|
||||
local_dir=str(AUTO_TRAIN_MODEL_DIR),
|
||||
allow_patterns=model_patterns,
|
||||
)
|
||||
)
|
||||
model = onnx_asr.load_model(
|
||||
DEFAULT_PARAKEET_ONNX_MODEL,
|
||||
str(snapshot_root),
|
||||
quantization=DEFAULT_PARAKEET_ONNX_QUANTIZATION,
|
||||
providers=list(providers),
|
||||
)
|
||||
finally:
|
||||
for key, value in previous.items():
|
||||
if value is None:
|
||||
os.environ.pop(key, None)
|
||||
else:
|
||||
os.environ[key] = value
|
||||
PARAKEET_ONNX_MODEL_CACHE.clear()
|
||||
PARAKEET_ONNX_MODEL_CACHE[cache_key] = model
|
||||
return model
|
||||
|
||||
|
||||
def _normalized_wav_float32(audio_path: Path):
|
||||
import numpy as np
|
||||
|
||||
with wave.open(str(audio_path), "rb") as wav_file:
|
||||
channels = wav_file.getnchannels()
|
||||
sample_width = wav_file.getsampwidth()
|
||||
sample_rate = wav_file.getframerate()
|
||||
frames = wav_file.readframes(wav_file.getnframes())
|
||||
if sample_width != 2 or sample_rate != TARGET_SAMPLE_RATE or channels < 1:
|
||||
raise RuntimeError("STT input must be 16 kHz, 16-bit PCM WAV audio.")
|
||||
samples = np.frombuffer(frames, dtype=np.int16).astype(np.float32)
|
||||
if channels > 1:
|
||||
samples = samples.reshape((-1, channels)).mean(axis=1)
|
||||
return samples / 32768.0
|
||||
|
||||
|
||||
def _transcribe_capture_with_parakeet(audio_path: Path, *, model: str, language: str) -> str:
|
||||
parakeet_model = _load_parakeet_onnx_model()
|
||||
kwargs: Dict[str, Any] = {
|
||||
"sample_rate": TARGET_SAMPLE_RATE,
|
||||
"channel": "mean",
|
||||
}
|
||||
if language:
|
||||
kwargs["language"] = language
|
||||
with PARAKEET_ONNX_TRANSCRIBE_LOCK:
|
||||
result = parakeet_model.recognize(
|
||||
_normalized_wav_float32(audio_path),
|
||||
**kwargs,
|
||||
)
|
||||
providers = _parakeet_onnx_providers()
|
||||
with AUTO_TRAIN_LOCK:
|
||||
AUTO_TRAIN_STATE["last_stt_engine"] = STT_ENGINE_PARAKEET_ONNX
|
||||
AUTO_TRAIN_STATE["last_stt_model"] = model
|
||||
AUTO_TRAIN_STATE["last_stt_device"] = providers[0]
|
||||
AUTO_TRAIN_STATE["last_stt_compute_type"] = DEFAULT_PARAKEET_ONNX_QUANTIZATION
|
||||
_save_auto_train_state_locked()
|
||||
return re.sub(r"\s+", " ", str(result or "")).strip()
|
||||
|
||||
|
||||
def _transcribe_capture(audio_path: Path, *, engine: str, language: str) -> str:
|
||||
token = _normalize_stt_engine(engine)
|
||||
model = _managed_stt_model(token, language)
|
||||
if token == STT_ENGINE_PARAKEET_ONNX:
|
||||
return _transcribe_capture_with_parakeet(
|
||||
audio_path,
|
||||
model=model,
|
||||
language=language,
|
||||
)
|
||||
return _transcribe_capture_with_faster_whisper(
|
||||
audio_path,
|
||||
model=model,
|
||||
language=language,
|
||||
)
|
||||
|
||||
|
||||
def _clear_stt_model_caches(*, keep_engine: str) -> None:
|
||||
token = _normalize_stt_engine(keep_engine)
|
||||
cleared = False
|
||||
if token != STT_ENGINE_FASTER_WHISPER:
|
||||
with FASTER_WHISPER_TRANSCRIBE_LOCK:
|
||||
with FASTER_WHISPER_MODEL_LOCK:
|
||||
cleared = bool(FASTER_WHISPER_MODEL_CACHE) or cleared
|
||||
FASTER_WHISPER_MODEL_CACHE.clear()
|
||||
if token != STT_ENGINE_PARAKEET_ONNX:
|
||||
with PARAKEET_ONNX_TRANSCRIBE_LOCK:
|
||||
with PARAKEET_ONNX_MODEL_LOCK:
|
||||
cleared = bool(PARAKEET_ONNX_MODEL_CACHE) or cleared
|
||||
PARAKEET_ONNX_MODEL_CACHE.clear()
|
||||
if cleared:
|
||||
gc.collect()
|
||||
|
||||
|
||||
def _queue_auto_review(file_name: str) -> bool:
|
||||
safe_file_name = Path(str(file_name or "")).name
|
||||
if not safe_file_name:
|
||||
@@ -884,7 +1177,7 @@ def _queue_pending_auto_reviews(*, force: bool = False) -> int:
|
||||
metadata.pop("auto_review_status", None)
|
||||
_write_sidecar_json(audio_path, metadata)
|
||||
status = ""
|
||||
if force and status in {"error", "no_speech"}:
|
||||
if force and status in {"error", "no_speech", "wake_phrase_ambiguous"}:
|
||||
metadata.pop("auto_review_status", None)
|
||||
_write_sidecar_json(audio_path, metadata)
|
||||
status = ""
|
||||
@@ -960,12 +1253,17 @@ def _auto_review_capture(file_name: str) -> None:
|
||||
metadata["auto_review_status"] = "transcribing"
|
||||
metadata["auto_reviewed_at"] = _iso_now()
|
||||
metadata["auto_review_wake_phrase"] = wake_phrase
|
||||
metadata["auto_review_stt_model"] = config["stt_model"]
|
||||
stt_engine = _normalize_stt_engine(config.get("stt_engine"))
|
||||
metadata["auto_review_stt_engine"] = stt_engine
|
||||
metadata["auto_review_stt_model"] = _managed_stt_model(
|
||||
stt_engine,
|
||||
config.get("language"),
|
||||
)
|
||||
_write_sidecar_json(audio_path, metadata)
|
||||
|
||||
transcript = _transcribe_capture_with_faster_whisper(
|
||||
transcript = _transcribe_capture(
|
||||
audio_path,
|
||||
model=str(config["stt_model"]),
|
||||
engine=stt_engine,
|
||||
language=str(config.get("language") or DEFAULT_LANGUAGE),
|
||||
)
|
||||
normalized = _normalize_transcript_text(transcript)
|
||||
@@ -980,11 +1278,38 @@ def _auto_review_capture(file_name: str) -> None:
|
||||
_record_auto_review_result(file_name=file_name, transcript=transcript, result="no_speech")
|
||||
return
|
||||
|
||||
if _transcript_contains_wake_phrase(transcript, wake_phrase):
|
||||
phrase_similarity = _wake_phrase_similarity(transcript, wake_phrase)
|
||||
phrase_detected = _transcript_contains_wake_phrase(transcript, wake_phrase)
|
||||
match_method = "exact" if phrase_detected else ""
|
||||
metadata["auto_review_phrase_similarity"] = round(phrase_similarity, 4)
|
||||
|
||||
if (
|
||||
not phrase_detected
|
||||
and phrase_similarity >= WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY
|
||||
and stt_engine == STT_ENGINE_FASTER_WHISPER
|
||||
):
|
||||
guided_transcript = _transcribe_capture_with_faster_whisper_guided(
|
||||
audio_path,
|
||||
model=str(metadata["auto_review_stt_model"]),
|
||||
language=str(config.get("language") or DEFAULT_LANGUAGE),
|
||||
wake_phrase=wake_phrase,
|
||||
)
|
||||
metadata["auto_review_guided_transcript"] = guided_transcript
|
||||
if _transcript_contains_wake_phrase(guided_transcript, wake_phrase):
|
||||
phrase_detected = True
|
||||
match_method = "guided_close_match"
|
||||
|
||||
if match_method:
|
||||
metadata["auto_review_match_method"] = match_method
|
||||
|
||||
if phrase_detected:
|
||||
guided_confirmation = match_method == "guided_close_match"
|
||||
if is_close_miss:
|
||||
metadata["auto_review_status"] = "approved_positive"
|
||||
metadata["auto_review_reason"] = (
|
||||
"Close miss contained the configured wake phrase and was promoted to a positive sample."
|
||||
"Close miss was confirmed as the configured wake phrase and promoted to a positive sample."
|
||||
if guided_confirmation
|
||||
else "Close miss contained the configured wake phrase and was promoted to a positive sample."
|
||||
)
|
||||
metadata["auto_positive"] = True
|
||||
_write_sidecar_json(audio_path, metadata)
|
||||
@@ -1009,11 +1334,29 @@ def _auto_review_capture(file_name: str) -> None:
|
||||
)
|
||||
return
|
||||
metadata["auto_review_status"] = "wake_phrase_detected"
|
||||
metadata["auto_review_reason"] = "Wake phrase found in transcript; left for manual positive review."
|
||||
metadata["auto_review_reason"] = (
|
||||
"Wake phrase confirmed by a guided second STT pass; left for manual positive review."
|
||||
if guided_confirmation
|
||||
else "Wake phrase found in transcript; left for manual positive review."
|
||||
)
|
||||
_write_sidecar_json(audio_path, metadata)
|
||||
_record_auto_review_result(file_name=file_name, transcript=transcript, result="wake_phrase_detected")
|
||||
return
|
||||
|
||||
if phrase_similarity >= WAKE_PHRASE_GUIDANCE_MIN_SIMILARITY:
|
||||
metadata["auto_review_status"] = "wake_phrase_ambiguous"
|
||||
metadata["auto_review_reason"] = (
|
||||
"STT sounded close to the configured wake phrase but could not confirm it; "
|
||||
"left for manual review."
|
||||
)
|
||||
_write_sidecar_json(audio_path, metadata)
|
||||
_record_auto_review_result(
|
||||
file_name=file_name,
|
||||
transcript=transcript,
|
||||
result="wake_phrase_ambiguous",
|
||||
)
|
||||
return
|
||||
|
||||
if is_close_miss:
|
||||
metadata["auto_review_status"] = "close_miss_phrase_not_detected"
|
||||
metadata["auto_review_reason"] = (
|
||||
@@ -1922,6 +2265,9 @@ def _captured_item_from_path(audio_path: Path) -> Dict[str, Any]:
|
||||
"auto_review_status": meta.get("auto_review_status") or "",
|
||||
"auto_review_reason": meta.get("auto_review_reason") or "",
|
||||
"auto_review_error": meta.get("auto_review_error") or "",
|
||||
"auto_review_guided_transcript": meta.get("auto_review_guided_transcript") or "",
|
||||
"auto_review_phrase_similarity": meta.get("auto_review_phrase_similarity"),
|
||||
"auto_review_match_method": meta.get("auto_review_match_method") or "",
|
||||
"size_bytes": stat.st_size,
|
||||
"audio_url": f"/api/audio/captured/{audio_path.name}",
|
||||
}
|
||||
@@ -2386,7 +2732,7 @@ def auto_train_status(request: Request):
|
||||
payload = _auto_train_status_payload()
|
||||
payload["ok"] = True
|
||||
payload["advertised_base_url"] = _advertised_base_url(request)
|
||||
payload["stt_backend"] = "faster-whisper"
|
||||
payload["stt_backend"] = payload["config"].get("stt_engine")
|
||||
return payload
|
||||
|
||||
|
||||
@@ -2420,6 +2766,8 @@ def update_auto_train(payload: Dict[str, Any] = None):
|
||||
)
|
||||
if schedule_changed or not AUTO_TRAIN_STATE.get("next_run_at"):
|
||||
_schedule_next_auto_run_locked()
|
||||
if previous.get("stt_engine") != normalized.get("stt_engine"):
|
||||
_clear_stt_model_caches(keep_engine=normalized["stt_engine"])
|
||||
if normalized["enabled"]:
|
||||
queued = _queue_pending_auto_reviews()
|
||||
AUTO_TRAIN_WAKE_EVENT.set()
|
||||
|
||||
Reference in New Issue
Block a user