Release NVIDIA WakeWord Trainer v23

VERSION

@@ -1 +1 @@
-22
+23

WHATS_NEW.md

@@ -1,19 +1,2 @@
-- Fixed the training console so scrolling up pauses auto-follow and provides a Jump to latest control.
-- Fixed trained wake-word cards and Copy URL to use the explicit JSON package URL instead of producing `undefined`.
-- Replaced the 128-profile clone pipeline with direct final-corpus generation from Qwen, OmniVoice, and Piper; MOSS now uses a different accepted carrier for each take.
-- Added strict provider-specific rejection for static, broadband/high-frequency noise, silence, clipping, excessive duration/rambling, and exact duplicate audio.
-- Capped Qwen and MOSS decoding for a single short utterance, fixed OmniVoice to bounded wake-phrase durations, and let safer providers fill every rejected share.
-- Expanded Qwen to 18,750 balanced combinations across gender, age, pitch, delivery, timbre, pace, and vocal weight before an instruction repeats.
-- Shifted the reactive trainer UI from blue-black surfaces to Tater's graphite-grey and orange visual theme.
-- Rebuilt the trainer interface as a reactive Vue 3 + TypeScript application using the same typed UI pattern as Tater.
-- Preserved session setup, multilingual TTS routing, sample review/import/trim, Auto Training, secure Tater pairing, live logs, and wake-word publishing in the new component-driven UI.
-- Updated the standard CUDA and Blackwell Dockerfiles to copy the complete prebuilt UI bundle; Node.js is not installed or required in the runtime image.
-- Made OmniVoice, Qwen3-TTS, MOSS-TTS-Nano, and Piper the recommended four-provider route where a compatible Piper model is present.
-- Added the live 646-language OmniVoice catalog, language quality tiers, exact per-engine routing, and normalized acoustic QA.
-- Added persistent per-engine environments and Hugging Face caches that keep conflicting TTS dependencies separate from wake-word training.
-- Improved automatic review accuracy for short wake phrases that STT initially hears as similar-sounding words.
-- Added a conservative Faster Whisper confirmation pass that uses the currently configured wake phrase only when the unbiased transcript is already phonetically close.
-- Kept unconfirmed close transcripts in the manual review inbox instead of allowing them to become harmful negative training samples.
-- Added visible guided-transcript and review-reason details, plus retry support for ambiguous clips through Review Now.
-- Locked the wake phrase, language, and TTS route while a session is active, and added Stop Session with clean full-process-tree training cancellation.
-- Added a Data tab with per-dataset disk usage and file counts, plus confirmed, training-safe deletion for recordings, downloads, generated caches, speech models, and training results.
+- Fixed NVIDIA v22 training runs remaining stuck immediately after Start Session instead of launching the training worker.
+- Corrected the worker-state handoff for both manual and automatic training, with regression coverage for the complete startup path.

tests/test_session_stop.py

@@ -1,7 +1,11 @@
 from __future__ import annotations
 
+import io
 import signal
+import tempfile
+import threading
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 
 import trainer_server as trainer
@@ -26,6 +30,19 @@ class _FakeTrainingProcess:
         self.returncode = -signal.SIGKILL
 
 
+class _CompletedTrainingProcess:
+    def __init__(self):
+        self.pid = 6543
+        self.returncode = 0
+        self.stdout = io.StringIO("worker started\n")
+
+    def poll(self):
+        return self.returncode
+
+    def wait(self, timeout=None):
+        return self.returncode
+
+
 class SessionStopTests(unittest.TestCase):
     def tearDown(self):
         trainer.TRAINING_STOP_EVENT.clear()
@@ -49,6 +66,51 @@ class SessionStopTests(unittest.TestCase):
             trainer.TRAINING_PROCESS = original_process
             trainer.TRAINING_THREAD = original_thread
 
+    def test_reserved_running_state_starts_the_background_worker(self):
+        original_process = trainer.TRAINING_PROCESS
+        original_thread = trainer.TRAINING_THREAD
+        original_raw_phrase = trainer.STATE.get("raw_phrase")
+        original_training = dict(trainer.STATE["training"])
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                data_dir = Path(directory)
+                process = _CompletedTrainingProcess()
+                trainer.TRAINING_PROCESS = None
+                trainer.TRAINING_THREAD = threading.current_thread()
+                with trainer.STATE_LOCK:
+                    trainer.STATE["raw_phrase"] = "hey tater"
+                    trainer.STATE["training"]["running"] = True
+
+                with (
+                    patch.object(trainer, "DATA_DIR", data_dir),
+                    patch.object(trainer, "_ensure_training_venv"),
+                    patch.object(trainer, "_ensure_training_datasets"),
+                    patch.object(trainer.subprocess, "Popen", return_value=process) as popen,
+                    patch.object(trainer, "_normalize_output_artifacts"),
+                ):
+                    trainer._run_training_background(
+                        "hey_tater",
+                        "en",
+                        True,
+                        auto_run=False,
+                        tts_mode="modern",
+                    )
+
+                popen.assert_called_once()
+                log_text = (data_dir / "recorder_training.log").read_text(encoding="utf-8")
+                self.assertIn("Nvidia Docker Training Run", log_text)
+                self.assertIn("worker started", log_text)
+                self.assertFalse(trainer.STATE["training"]["running"])
+                self.assertEqual(trainer.STATE["training"]["exit_code"], 0)
+                self.assertIsNone(trainer.TRAINING_THREAD)
+        finally:
+            with trainer.STATE_LOCK:
+                trainer.STATE["raw_phrase"] = original_raw_phrase
+                trainer.STATE["training"].clear()
+                trainer.STATE["training"].update(original_training)
+            trainer.TRAINING_PROCESS = original_process
+            trainer.TRAINING_THREAD = original_thread
+
 
 if __name__ == "__main__":
     unittest.main()

trainer_server.py

@@ -3110,8 +3110,9 @@ def _run_training_background(
 
     with DATA_MANAGEMENT_LOCK:
         with STATE_LOCK:
-            if STATE["training"]["running"]:
-                return JSONResponse({"ok": False, "error": "Training already running"}, status_code=400)
+            # The API or auto-training scheduler reserves the run by setting
+            # this flag before the thread starts. Duplicate starts are already
+            # rejected there and by _start_training_thread's runtime lock.
             STATE["training"]["running"] = True
         STATE["training"]["exit_code"] = None
         STATE["training"]["log_lines"] = []