Release NVIDIA WakeWord Trainer v17

README.md

@@ -22,7 +22,7 @@ docker pull ghcr.io/tatertotterson/microwakeword:latest
 Tagged releases also publish matching immutable image tags:
 
 ```bash
-docker pull ghcr.io/tatertotterson/microwakeword:v15
+docker pull ghcr.io/tatertotterson/microwakeword:v17
 ```
 
 The release tag must match `VERSION`. Update `WHATS_NEW.md` before tagging; the Docker workflow prepends it to GitHub's automatically generated release notes.
@@ -32,7 +32,7 @@ Python 3.13 TensorFlow build for `sm_120`:
 
 ```bash
 docker pull ghcr.io/tatertotterson/microwakeword:blackwell
-docker pull ghcr.io/tatertotterson/microwakeword:v15-blackwell
+docker pull ghcr.io/tatertotterson/microwakeword:v17-blackwell
 ```
 
 Use the Blackwell image only for RTX 50-series cards. It includes the
@@ -53,9 +53,9 @@ docker run -d \
   ghcr.io/tatertotterson/microwakeword:latest
 ```
 
-Use a version tag such as `ghcr.io/tatertotterson/microwakeword:v15` when you want to pin a known release instead of tracking `latest`.
+Use a version tag such as `ghcr.io/tatertotterson/microwakeword:v17` when you want to pin a known release instead of tracking `latest`.
 For RTX 50-series cards, use `ghcr.io/tatertotterson/microwakeword:blackwell`
-or a pinned tag such as `ghcr.io/tatertotterson/microwakeword:v15-blackwell`
+or a pinned tag such as `ghcr.io/tatertotterson/microwakeword:v17-blackwell`
 in the same `docker run` command.
 
 The flags:

VERSION

@@ -1 +1 @@
-15
+17

WHATS_NEW.md

@@ -1,3 +1,2 @@
-- Added secure Tater linking: enter the short-lived code from Tater Voice Settings instead of giving the trainer a general API token.
+- Fixed the training-status endpoint crashing after a training log was created because its log-tail limits were missing.
-- Automatic and manual publishing now tell Tater which trained wake word is active, and Tater applies it globally to every connected satellite.
+- Restored bounded, incremental training-log updates so the UI can continue showing live progress without repeatedly reading the entire log.
-- Added clear linked, unlinked, and pairing-success states to the Auto Training interface.

run.sh

@@ -107,19 +107,33 @@ fi
 # Faster Whisper/CTranslate2 loads these CUDA libraries before Python starts.
 # They live in the persistent UI venv so both Docker image variants can use GPU STT.
 WHISPER_CUDA_LIBRARY_PATH="$("${PY}" - <<'PY'
-import os
+from importlib.util import find_spec
+from pathlib import Path
 
-try:
+
-    import nvidia.cublas.lib
+def package_directory(name):
-    import nvidia.cudnn.lib
+    try:
-except ImportError:
+        spec = find_spec(name)
-    print("")
+    except (ImportError, AttributeError, ValueError):
-else:
+        return ""
-    print(
+    if spec is None:
-        os.path.dirname(nvidia.cublas.lib.__file__)
+        return ""
-        + ":"
+
-        + os.path.dirname(nvidia.cudnn.lib.__file__)
+    for location in spec.submodule_search_locations or ():
-    )
+        if location:
+            return str(Path(location).resolve())
+
+    origin = spec.origin
+    if origin and origin not in {"built-in", "frozen"}:
+        return str(Path(origin).resolve().parent)
+    return ""
+
+
+paths = [
+    package_directory("nvidia.cublas.lib"),
+    package_directory("nvidia.cudnn.lib"),
+]
+print(":".join(dict.fromkeys(path for path in paths if path)))
 PY
 )"
 if [[ -n "${WHISPER_CUDA_LIBRARY_PATH}" ]]; then

tests/test_auto_train.py

@@ -425,6 +425,39 @@ class AutoTrainTests(unittest.TestCase):
         self.assertEqual(trainer.AUTO_TRAIN_STATE["last_stt_device"], "cuda")
         self.assertEqual(trainer.AUTO_TRAIN_STATE["last_stt_compute_type"], "float16")
 
+    def test_train_status_reads_and_increments_training_log_tail(self):
+        log_path = Path(self.tempdir.name) / "training.log"
+        log_path.write_text("first\nsecond\nthird\n", encoding="utf-8")
+        with trainer.STATE_LOCK:
+            original_training = dict(trainer.STATE["training"])
+            trainer.STATE["training"].update(
+                {
+                    "log_path": str(log_path),
+                    "last_sent_tail": [],
+                    "last_log_size": 0,
+                }
+            )
+
+        try:
+            with (
+                patch.object(trainer, "TRAIN_LOG_TAIL_LINES", 2),
+                patch.object(trainer, "TRAIN_LOG_MAX_BYTES", 1024),
+            ):
+                first_status = trainer.train_status()
+                self.assertEqual(first_status["training"]["log_lines"], ["second", "third"])
+                self.assertEqual(first_status["training"]["log_text"], "second\nthird")
+
+                with log_path.open("a", encoding="utf-8") as log_file:
+                    log_file.write("fourth\n")
+
+                next_status = trainer.train_status()
+                self.assertEqual(next_status["training"]["log_lines"], ["third", "fourth"])
+                self.assertEqual(next_status["training"]["log_text"], "fourth")
+        finally:
+            with trainer.STATE_LOCK:
+                trainer.STATE["training"].clear()
+                trainer.STATE["training"].update(original_training)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_run_sh.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+RUN_SH = REPO_ROOT / "run.sh"
+
+
+def _cuda_path_probe() -> str:
+    source = RUN_SH.read_text(encoding="utf-8")
+    match = re.search(
+        r'WHISPER_CUDA_LIBRARY_PATH="\$\("\$\{PY\}" - <<\'PY\'\n(?P<probe>.*?)\nPY\n\)"',
+        source,
+        flags=re.DOTALL,
+    )
+    if match is None:
+        raise AssertionError("Could not locate the CUDA library path probe in run.sh")
+    return match.group("probe")
+
+
+class RunShCudaLibraryPathTests(unittest.TestCase):
+    def _run_probe(self, python_path: Path) -> subprocess.CompletedProcess[str]:
+        env = dict(os.environ)
+        env["PYTHONPATH"] = str(python_path)
+        return subprocess.run(
+            [sys.executable, "-S", "-"],
+            input=_cuda_path_probe(),
+            text=True,
+            capture_output=True,
+            check=False,
+            env=env,
+        )
+
+    def test_namespace_cuda_packages_do_not_require_module_file(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            root = Path(temp_dir)
+            cublas_lib = root / "nvidia" / "cublas" / "lib"
+            cudnn_lib = root / "nvidia" / "cudnn" / "lib"
+            cublas_lib.mkdir(parents=True)
+            cudnn_lib.mkdir(parents=True)
+
+            result = self._run_probe(root)
+
+            self.assertEqual(result.returncode, 0, result.stderr)
+            self.assertEqual(
+                result.stdout.strip().split(":"),
+                [str(cublas_lib.resolve()), str(cudnn_lib.resolve())],
+            )
+
+    def test_missing_cuda_packages_return_an_empty_path(self) -> None:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            result = self._run_probe(Path(temp_dir))
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(result.stdout.strip(), "")
+
+
+if __name__ == "__main__":
+    unittest.main()

trainer_server.py

@@ -70,6 +70,8 @@ PIPER_CATALOG_CACHE_FILE = Path(
         str(DATA_DIR / ".cache" / "piper_voices_catalog.json"),
     )
 ).resolve()
+TRAIN_LOG_TAIL_LINES = int(os.environ.get("REC_TRAIN_LOG_TAIL_LINES", "400"))
+TRAIN_LOG_MAX_BYTES = int(os.environ.get("REC_TRAIN_LOG_MAX_BYTES", str(512 * 1024)))
 
 DATASET_CLEANUP_ARCHIVES = os.environ.get("REC_DATASET_CLEANUP_ARCHIVES", "false").lower() in ("1", "true", "yes", "y")
 DATASET_CLEANUP_INTERMEDIATE = os.environ.get("REC_DATASET_CLEANUP_INTERMEDIATE_FILES", "false").lower() in ("1", "true", "yes", "y")