diff --git a/VERSION b/VERSION index 2bd5a0a..4099407 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -22 +23 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 58c5bbb..17bd332 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,19 +1,2 @@ -- Fixed the training console so scrolling up pauses auto-follow and provides a Jump to latest control. -- Fixed trained wake-word cards and Copy URL to use the explicit JSON package URL instead of producing `undefined`. -- Replaced the 128-profile clone pipeline with direct final-corpus generation from Qwen, OmniVoice, and Piper; MOSS now uses a different accepted carrier for each take. -- Added strict provider-specific rejection for static, broadband/high-frequency noise, silence, clipping, excessive duration/rambling, and exact duplicate audio. -- Capped Qwen and MOSS decoding for a single short utterance, fixed OmniVoice to bounded wake-phrase durations, and let safer providers fill every rejected share. -- Expanded Qwen to 18,750 balanced combinations across gender, age, pitch, delivery, timbre, pace, and vocal weight before an instruction repeats. -- Shifted the reactive trainer UI from blue-black surfaces to Tater's graphite-grey and orange visual theme. -- Rebuilt the trainer interface as a reactive Vue 3 + TypeScript application using the same typed UI pattern as Tater. -- Preserved session setup, multilingual TTS routing, sample review/import/trim, Auto Training, secure Tater pairing, live logs, and wake-word publishing in the new component-driven UI. -- Updated the standard CUDA and Blackwell Dockerfiles to copy the complete prebuilt UI bundle; Node.js is not installed or required in the runtime image. -- Made OmniVoice, Qwen3-TTS, MOSS-TTS-Nano, and Piper the recommended four-provider route where a compatible Piper model is present. -- Added the live 646-language OmniVoice catalog, language quality tiers, exact per-engine routing, and normalized acoustic QA. -- Added persistent per-engine environments and Hugging Face caches that keep conflicting TTS dependencies separate from wake-word training. -- Improved automatic review accuracy for short wake phrases that STT initially hears as similar-sounding words. -- Added a conservative Faster Whisper confirmation pass that uses the currently configured wake phrase only when the unbiased transcript is already phonetically close. -- Kept unconfirmed close transcripts in the manual review inbox instead of allowing them to become harmful negative training samples. -- Added visible guided-transcript and review-reason details, plus retry support for ambiguous clips through Review Now. -- Locked the wake phrase, language, and TTS route while a session is active, and added Stop Session with clean full-process-tree training cancellation. -- Added a Data tab with per-dataset disk usage and file counts, plus confirmed, training-safe deletion for recordings, downloads, generated caches, speech models, and training results. +- Fixed NVIDIA v22 training runs remaining stuck immediately after Start Session instead of launching the training worker. +- Corrected the worker-state handoff for both manual and automatic training, with regression coverage for the complete startup path. diff --git a/tests/test_session_stop.py b/tests/test_session_stop.py index e943427..a523605 100644 --- a/tests/test_session_stop.py +++ b/tests/test_session_stop.py @@ -1,7 +1,11 @@ from __future__ import annotations +import io import signal +import tempfile +import threading import unittest +from pathlib import Path from unittest.mock import patch import trainer_server as trainer @@ -26,6 +30,19 @@ class _FakeTrainingProcess: self.returncode = -signal.SIGKILL +class _CompletedTrainingProcess: + def __init__(self): + self.pid = 6543 + self.returncode = 0 + self.stdout = io.StringIO("worker started\n") + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + return self.returncode + + class SessionStopTests(unittest.TestCase): def tearDown(self): trainer.TRAINING_STOP_EVENT.clear() @@ -49,6 +66,51 @@ class SessionStopTests(unittest.TestCase): trainer.TRAINING_PROCESS = original_process trainer.TRAINING_THREAD = original_thread + def test_reserved_running_state_starts_the_background_worker(self): + original_process = trainer.TRAINING_PROCESS + original_thread = trainer.TRAINING_THREAD + original_raw_phrase = trainer.STATE.get("raw_phrase") + original_training = dict(trainer.STATE["training"]) + try: + with tempfile.TemporaryDirectory() as directory: + data_dir = Path(directory) + process = _CompletedTrainingProcess() + trainer.TRAINING_PROCESS = None + trainer.TRAINING_THREAD = threading.current_thread() + with trainer.STATE_LOCK: + trainer.STATE["raw_phrase"] = "hey tater" + trainer.STATE["training"]["running"] = True + + with ( + patch.object(trainer, "DATA_DIR", data_dir), + patch.object(trainer, "_ensure_training_venv"), + patch.object(trainer, "_ensure_training_datasets"), + patch.object(trainer.subprocess, "Popen", return_value=process) as popen, + patch.object(trainer, "_normalize_output_artifacts"), + ): + trainer._run_training_background( + "hey_tater", + "en", + True, + auto_run=False, + tts_mode="modern", + ) + + popen.assert_called_once() + log_text = (data_dir / "recorder_training.log").read_text(encoding="utf-8") + self.assertIn("Nvidia Docker Training Run", log_text) + self.assertIn("worker started", log_text) + self.assertFalse(trainer.STATE["training"]["running"]) + self.assertEqual(trainer.STATE["training"]["exit_code"], 0) + self.assertIsNone(trainer.TRAINING_THREAD) + finally: + with trainer.STATE_LOCK: + trainer.STATE["raw_phrase"] = original_raw_phrase + trainer.STATE["training"].clear() + trainer.STATE["training"].update(original_training) + trainer.TRAINING_PROCESS = original_process + trainer.TRAINING_THREAD = original_thread + if __name__ == "__main__": unittest.main() diff --git a/trainer_server.py b/trainer_server.py index eac153e..71516e9 100644 --- a/trainer_server.py +++ b/trainer_server.py @@ -3110,8 +3110,9 @@ def _run_training_background( with DATA_MANAGEMENT_LOCK: with STATE_LOCK: - if STATE["training"]["running"]: - return JSONResponse({"ok": False, "error": "Training already running"}, status_code=400) + # The API or auto-training scheduler reserves the run by setting + # this flag before the thread starts. Duplicate starts are already + # rejected there and by _start_training_thread's runtime lock. STATE["training"]["running"] = True STATE["training"]["exit_code"] = None STATE["training"]["log_lines"] = []