diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1bf2c40..3770b12 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -26,6 +26,18 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Validate tag matches trainer version + if: startsWith(github.ref, 'refs/tags/') + shell: bash + run: | + set -euo pipefail + version="$(tr -d '[:space:]' < VERSION)" + expected_tag="v${version#v}" + if [[ "${GITHUB_REF_NAME}" != "${expected_tag}" ]]; then + echo "Tag ${GITHUB_REF_NAME} does not match trainer version ${expected_tag}." >&2 + exit 1 + fi + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -94,6 +106,7 @@ jobs: title="microWakeWord Nvidia Trainer ${TAG_NAME}" generated_notes="$(mktemp)" release_notes="$(mktemp)" + test -s WHATS_NEW.md gh api "repos/${REPO}/releases/generate-notes" \ -f tag_name="${TAG_NAME}" \ @@ -101,6 +114,11 @@ jobs: --jq '.body' > "${generated_notes}" { + echo "## What's New" + echo + cat WHATS_NEW.md + echo + echo echo "## Docker Images" echo echo "- \`ghcr.io/tatertotterson/microwakeword:${TAG_NAME}\`" diff --git a/README.md b/README.md index e70f264..6c3116e 100644 --- a/README.md +++ b/README.md @@ -22,15 +22,17 @@ docker pull ghcr.io/tatertotterson/microwakeword:latest Tagged releases also publish matching immutable image tags: ```bash -docker pull ghcr.io/tatertotterson/microwakeword:v11 +docker pull ghcr.io/tatertotterson/microwakeword:v12 ``` +The release tag must match `VERSION`. Update `WHATS_NEW.md` before tagging; the Docker workflow prepends it to GitHub's automatically generated release notes. + RTX 50-series / Blackwell GPUs use a separate image with CUDA 12.8 and a Python 3.13 TensorFlow build for `sm_120`: ```bash docker pull ghcr.io/tatertotterson/microwakeword:blackwell -docker pull ghcr.io/tatertotterson/microwakeword:v11-blackwell +docker pull ghcr.io/tatertotterson/microwakeword:v12-blackwell ``` Use the Blackwell image only for RTX 50-series cards. It includes the @@ -51,9 +53,9 @@ docker run -d \ ghcr.io/tatertotterson/microwakeword:latest ``` -Use a version tag such as `ghcr.io/tatertotterson/microwakeword:v11` when you want to pin a known release instead of tracking `latest`. +Use a version tag such as `ghcr.io/tatertotterson/microwakeword:v12` when you want to pin a known release instead of tracking `latest`. For RTX 50-series cards, use `ghcr.io/tatertotterson/microwakeword:blackwell` -or a pinned tag such as `ghcr.io/tatertotterson/microwakeword:v11-blackwell` +or a pinned tag such as `ghcr.io/tatertotterson/microwakeword:v12-blackwell` in the same `docker run` command. The flags: @@ -78,6 +80,7 @@ If you change `REC_PORT`, open that port instead and use the same port in the sa ## What The UI Does - `Trainer` starts a wake-word session, shows positive/negative sample counts, and launches training. +- `Auto Training` transcribes real wake triggers, promotes phrase-misses to hard negatives, schedules retraining, and refreshes Tater Native satellites. - `Captured Audio` reviews clips sent by Tater Native or ESPHome sats, including wake hits, close misses, and false wakes. - `Samples` plays, removes, clears, and manually imports personal or negative samples. - `Wake Words` lists locally trained JSON/model links for live wake-word switching in Tater. @@ -162,6 +165,27 @@ Starting a new session does not clear samples. Use the clear buttons in `Samples --- +## Auto Training + +`Auto Training` is an opt-in false-positive loop. It is disabled until you enter the exact wake phrase and enable it. + +For each new wake-trigger clip sent to the trainer: + +1. Faster Whisper transcribes the audio locally. +2. If the transcript contains the configured wake phrase, the clip stays in `Captured Audio` for manual positive review. +3. If speech was transcribed but the wake phrase is absent, the clip moves to `/data/negative_samples/` as an auto-reviewed hard negative. +4. Empty transcripts, close misses, VAD-blocked captures, and captures for another wake word stay out of the automatic negative path. + +The default `small.en` model uses CUDA with `float16` when CTranslate2 can see an NVIDIA GPU, and falls back to CPU with `int8`. Choose a multilingual Faster Whisper model such as `small` when the wake phrase is not English. Downloaded STT models are cached in `/data/auto_train_models/`. + +Scheduled training runs only after the configured number of new automatic negatives has accumulated. A successful run publishes the replacement model at the same wake-word URL and can call Tater's native satellite settings API to make connected satellites pull it again. This refresh uses the existing Tater Native update path, so no satellite firmware change is required. + +The `Trainer public URL` must be reachable from the satellites. With the documented `--network host` command, the trainer can normally use the LAN address from the browser request or host network. If you open the UI as `http://localhost:8789`, enter a value such as `http://192.168.1.50:8789`, or start the container with `REC_PUBLIC_BASE_URL` set to that value. When using Docker bridge networking, always set this URL to the published host address; a container bridge address is not satellite-reachable. + +The default Tater URL, `http://127.0.0.1:8501`, assumes the documented host networking. Change it to a container-reachable Tater address if you use another Docker network. The optional API token is stored in `/data/auto_train_config.json` with owner-only permissions. + +--- + ## Training Flow 1. Enter the wake phrase in `Trainer`. @@ -212,6 +236,7 @@ After those assets are prepared, later runs reuse the local copies unless the mo The `Wake Words` tab lists locally trained wake-word packages from `/data/trained_wake_words/`. - Copy the JSON URL into the Tater Native satellite settings to switch wake words live. +- Links use the configured public trainer URL, a non-loopback browser host, or the detected LAN address instead of advertising `127.0.0.1` to satellites. - Open the JSON or model links directly for quick inspection. - The JSON includes the matching model path plus Tater tuning metadata. - No firmware flashing happens from this trainer app anymore. @@ -244,7 +269,7 @@ The JSON keeps the standard microWakeWord fields for compatibility: { "micro": { "probability_cutoff": 0.97, - "sliding_window_size": 5 + "sliding_window_size": 6 } } ``` @@ -259,8 +284,8 @@ It also includes Tater Native metadata used by newer satellites and the Tater se "tater_native": { "format_version": 1, "wake_threshold": 0.97, - "wake_sliding_window": 5, - "close_miss_threshold": 0.78, + "wake_sliding_window": 6, + "close_miss_threshold": 0.80, "frontend": { "name": "tflm_microfrontend", "sample_rate": 16000, @@ -273,6 +298,7 @@ It also includes Tater Native metadata used by newer satellites and the Tater se ``` Calibration metrics are included under `calibration` so false accepts/hour and recall can be surfaced in the UI. +Calibration evaluates thresholds from `0.95` through `1.00` with sliding windows of `5`, `6`, and `7`. Among candidates within 0.5 percentage points of the best recall, it prefers the lowest measured ambient false-accept rate. If calibration cannot complete, packaging uses the conservative `0.97` threshold and a window of `6`. --- @@ -289,6 +315,7 @@ That removes: - cached datasets - training environments - trained models +- Auto Training settings, state, transcripts, and cached Faster Whisper models --- @@ -296,6 +323,7 @@ That removes: - Personal samples are optional. - Negative samples are optional but useful for reducing false wakes. +- Auto Training is disabled by default and only classifies actual wake triggers automatically. - The UI server is `trainer_server.py`. - The launcher is `run.sh`. - Trainer capture settings live in Tater for Tater Native satellites, and on device entities for older ESPHome satellites. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..48082f7 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +12 diff --git a/WHATS_NEW.md b/WHATS_NEW.md new file mode 100644 index 0000000..b37f98c --- /dev/null +++ b/WHATS_NEW.md @@ -0,0 +1,5 @@ +- Added opt-in Auto Training for false-positive wake triggers, using Faster Whisper with automatic CUDA/float16 selection and CPU/int8 fallback. +- Wake triggers whose transcripts do not contain the configured phrase can now become hard negatives automatically; close misses, empty transcripts, and phrase matches remain available for manual review. +- Added scheduled retraining with a minimum-new-negatives threshold and automatic Tater Native satellite refresh after a successful model build. +- Wake-word download links now advertise a LAN-reachable trainer URL instead of `127.0.0.1`. +- Tightened detector calibration defaults to favor fewer ambient false accepts while preserving candidates within 0.5 percentage points of the best recall. diff --git a/cli/calibrate_detector.py b/cli/calibrate_detector.py index aad90d9..9674377 100644 --- a/cli/calibrate_detector.py +++ b/cli/calibrate_detector.py @@ -9,24 +9,22 @@ import math import os from datetime import datetime, timezone from pathlib import Path -from typing import Iterable, Sequence +from typing import Any, Iterable, Sequence import numpy as np import yaml -from microwakeword.data import FeatureHandler -from microwakeword.inference import Model - - -DEFAULT_WINDOW_SIZES = [4, 5, 6, 7] +DEFAULT_WINDOW_SIZES = [5, 6, 7] DEFAULT_TARGET_FAPH = float(os.environ.get("MWW_CALIBRATION_TARGET_FAPH", "0.25")) DEFAULT_COOLDOWN_SLICES = int(os.environ.get("MWW_CALIBRATION_COOLDOWN_SLICES", "25")) DEFAULT_POSITIVE_SKIP_SLICES = int( os.environ.get("MWW_CALIBRATION_POSITIVE_SKIP_SLICES", "25") ) DEFAULT_CUTOFF_STEP = float(os.environ.get("MWW_CALIBRATION_CUTOFF_STEP", "0.01")) -DEFAULT_CUTOFF_MIN = float(os.environ.get("MWW_CALIBRATION_CUTOFF_MIN", "0.85")) +DEFAULT_CUTOFF_MIN = float(os.environ.get("MWW_CALIBRATION_CUTOFF_MIN", "0.95")) DEFAULT_CUTOFF_MAX = float(os.environ.get("MWW_CALIBRATION_CUTOFF_MAX", "1.00")) +DEFAULT_RECALL_MARGIN = float(os.environ.get("MWW_CALIBRATION_RECALL_MARGIN", "0.005")) +PREFERRED_WINDOW_SIZE = 6 def parse_args() -> argparse.Namespace: @@ -65,6 +63,15 @@ def parse_args() -> argparse.Namespace: default=DEFAULT_TARGET_FAPH, help="Target ambient false accepts per hour for the selected operating point.", ) + parser.add_argument( + "--recall-margin", + type=float, + default=DEFAULT_RECALL_MARGIN, + help=( + "Maximum recall loss allowed when preferring a candidate with fewer " + "ambient false accepts (0.005 means 0.5 percentage points)." + ), + ) parser.add_argument( "--cooldown-slices", type=int, @@ -159,7 +166,13 @@ def _compute_false_accepts_per_hour( def _select_best_candidate( candidates: list[dict[str, float]], target_faph: float, + recall_margin: float = DEFAULT_RECALL_MARGIN, ) -> tuple[dict[str, float], float]: + if not candidates: + raise ValueError("at least one calibration candidate is required") + if recall_margin < 0: + raise ValueError("recall margin must be >= 0") + fallback_limits = [ target_faph, max(target_faph * 2.0, target_faph + 0.5), @@ -172,13 +185,27 @@ def _select_best_candidate( return index return len(fallback_limits) + # Stay in the strictest false-accept tier that has a viable candidate. Within + # that tier, keep candidates close to the best recall, then spend the allowed + # recall margin on the lowest measured false-accept rate. + best_tier = min(tier(candidate) for candidate in candidates) + tier_candidates = [ + candidate for candidate in candidates if tier(candidate) == best_tier + ] + best_recall = max(candidate["recall"] for candidate in tier_candidates) + recall_floor = best_recall - recall_margin + viable_candidates = [ + candidate + for candidate in tier_candidates + if candidate["recall"] >= recall_floor - 1e-12 + ] + best = min( - candidates, + viable_candidates, key=lambda candidate: ( - tier(candidate), - -candidate["recall"], candidate["false_accepts_per_hour"], - abs(candidate["sliding_window_size"] - 5), + -candidate["recall"], + abs(candidate["sliding_window_size"] - PREFERRED_WINDOW_SIZE), -candidate["probability_cutoff"], ), ) @@ -195,7 +222,7 @@ def _load_config(config_path: Path) -> dict: def _load_eval_sets( - handler: FeatureHandler, + handler: Any, config: dict, ) -> tuple[str, str, list[np.ndarray], list[np.ndarray]]: for positive_mode, ambient_mode in ( @@ -228,7 +255,7 @@ def _load_eval_sets( def _predict_tracks( - model: Model, + model: Any, tracks: Sequence[np.ndarray], label: str, ) -> list[np.ndarray]: @@ -244,8 +271,13 @@ def _predict_tracks( def main() -> int: + from microwakeword.data import FeatureHandler + from microwakeword.inference import Model + args = parse_args() window_sizes = _parse_window_sizes(args.window_sizes) + if args.recall_margin < 0 or args.recall_margin > 1: + raise ValueError("recall-margin must be between 0 and 1") if args.cutoff_step <= 0: raise ValueError("cutoff-step must be > 0") if args.cutoff_max < args.cutoff_min: @@ -276,6 +308,10 @@ def main() -> int: f"→ Evaluating window sizes {window_sizes} with target <= " f"{args.target_faph:.2f} false accepts/hour" ) + print( + f"→ Favoring lower false accepts within " + f"{args.recall_margin:.2%} of the best recall" + ) config = _load_config(config_path) config["flags"] = config.get("flags", {}) @@ -338,7 +374,11 @@ def main() -> int: candidates.append(candidate) window_candidates.append(candidate) - best_window, _ = _select_best_candidate(window_candidates, args.target_faph) + best_window, _ = _select_best_candidate( + window_candidates, + args.target_faph, + args.recall_margin, + ) best_by_window.append(best_window) print( " window={window}: cutoff={cutoff:.2f}; recall={recall:.2%}; " @@ -350,7 +390,11 @@ def main() -> int: ) ) - best, selected_limit = _select_best_candidate(candidates, args.target_faph) + best, selected_limit = _select_best_candidate( + candidates, + args.target_faph, + args.recall_margin, + ) if best["false_accepts_per_hour"] > args.target_faph + 1e-9: print( "⚠️ No candidate met the target false accepts/hour budget; " @@ -390,6 +434,8 @@ def main() -> int: "cutoff_min": round(float(cutoffs[0]), 4), "cutoff_max": round(float(cutoffs[-1]), 4), "cutoff_step": float(args.cutoff_step), + "recall_margin": float(args.recall_margin), + "preferred_window_size": PREFERRED_WINDOW_SIZE, }, "per_window_best": best_by_window, "generated_at": datetime.now(timezone.utc).isoformat(), diff --git a/cli/wake_word_sample_trainer b/cli/wake_word_sample_trainer index 1719ed1..00f7fed 100644 --- a/cli/wake_word_sample_trainer +++ b/cli/wake_word_sample_trainer @@ -467,7 +467,12 @@ echo "🎯 Calibrating detector settings for on-device use…" if "${PYTHON_BIN:-python}" "${PROGDIR}/calibrate_detector.py" \ --training-config "${WORK_DIR}/trained_models/wakeword/training_config.yaml" \ --model "${source_path}" \ - --output "${calibration_path}"; then + --output "${calibration_path}" \ + --target-faph "${MWW_CALIBRATION_TARGET_FAPH:-0.25}" \ + --recall-margin "${MWW_CALIBRATION_RECALL_MARGIN:-0.005}" \ + --window-sizes "${MWW_CALIBRATION_WINDOW_SIZES:-5,6,7}" \ + --cutoff-min "${MWW_CALIBRATION_CUTOFF_MIN:-0.95}" \ + --cutoff-max "${MWW_CALIBRATION_CUTOFF_MAX:-1.00}"; then echo "✅ Detector calibration complete." else echo "⚠️ Detector calibration failed; packaging with default detector settings." @@ -496,8 +501,8 @@ from pathlib import Path json_path = Path(os.environ["JSON_PATH"]) calibration_path = Path(os.environ.get("CALIBRATION_PATH", "")) language = (os.environ.get("LANGUAGE", "en") or "en").strip().lower() -probability_cutoff = 0.85 -sliding_window_size = 4 +probability_cutoff = 0.97 +sliding_window_size = 6 strict_min_close_miss_threshold = 0.68 calibration = {} diff --git a/dockerfile b/dockerfile index ef4478e..c1821db 100644 --- a/dockerfile +++ b/dockerfile @@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND=noninteractive # System deps RUN apt-get update && apt-get install -y --no-install-recommends \ python3.12 python3.12-venv python3.12-dev python3-pip python-is-python3 \ - git wget curl unzip patch ninja-build ca-certificates nano less \ + git wget curl unzip patch ninja-build ca-certificates nano less libgomp1 \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /data diff --git a/dockerfile.blackwell b/dockerfile.blackwell index 5562553..78dce55 100644 --- a/dockerfile.blackwell +++ b/dockerfile.blackwell @@ -13,7 +13,7 @@ ENV MWW_BLACKWELL_TF_WHEEL_URL=https://github.com/chivitiH/tensorflow-blackwell- # Python 3.13 is used only for the Blackwell TensorFlow training step. RUN apt-get update && apt-get install -y --no-install-recommends \ software-properties-common ca-certificates curl git wget unzip patch \ - ninja-build nano less \ + ninja-build nano less libgomp1 \ && add-apt-repository -y ppa:deadsnakes/ppa \ && apt-get update \ && apt-get install -y --no-install-recommends \ diff --git a/run.sh b/run.sh index ea9f41f..321a782 100644 --- a/run.sh +++ b/run.sh @@ -31,7 +31,10 @@ install_ui_deps() { "uvicorn[standard]==${UVICORN_VERSION}" \ "python-multipart==${PY_MULTIPART_VERSION}" \ "silero-vad>=5.0.0" \ - "numpy>=1.24.0" + "numpy>=1.24.0" \ + "faster-whisper>=1.0.0" \ + "nvidia-cublas-cu12" \ + "nvidia-cudnn-cu12==9.*" } # ----------------------------- @@ -78,8 +81,13 @@ exact = { minimum = { "silero-vad": "5.0.0", "numpy": "1.24.0", + "faster-whisper": "1.0.0", + "nvidia-cudnn-cu12": "9.0.0", } -present = ("torch",) +present = ( + "torch", + "nvidia-cublas-cu12", +) for package, expected in exact.items(): if md.version(package) != expected: @@ -95,6 +103,28 @@ PY install_ui_deps fi fi + +# Faster Whisper/CTranslate2 loads these CUDA libraries before Python starts. +# They live in the persistent UI venv so both Docker image variants can use GPU STT. +WHISPER_CUDA_LIBRARY_PATH="$("${PY}" - <<'PY' +import os + +try: + import nvidia.cublas.lib + import nvidia.cudnn.lib +except ImportError: + print("") +else: + print( + os.path.dirname(nvidia.cublas.lib.__file__) + + ":" + + os.path.dirname(nvidia.cudnn.lib.__file__) + ) +PY +)" +if [[ -n "${WHISPER_CUDA_LIBRARY_PATH}" ]]; then + export LD_LIBRARY_PATH="${WHISPER_CUDA_LIBRARY_PATH}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" +fi # ----------------------------- # Trainer server env # ----------------------------- diff --git a/static/index.html b/static/index.html index d95bf0a..ddd4191 100644 --- a/static/index.html +++ b/static/index.html @@ -210,6 +210,35 @@ width: 100%; } + .autoGrid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 14px; + } + .autoGrid .wide { grid-column: 1 / -1; } + .checkField { + display: flex; + align-items: flex-start; + gap: 10px; + padding: 12px; + border-radius: 12px; + border: 1px solid rgba(255,255,255,0.09); + background: rgba(255,255,255,0.035); + color: var(--muted); + line-height: 1.4; + } + .checkField input { margin-top: 3px; } + .checkField[hidden] { display: none !important; } + .checkField strong { color: var(--text); display: block; margin-bottom: 3px; } + .autoActions { display: flex; flex-wrap: wrap; gap: 10px; } + .autoAudit { + padding: 14px; + border-radius: 14px; + border: 1px solid rgba(255,255,255,0.08); + background: rgba(0,0,0,0.22); + overflow-wrap: anywhere; + } + .firmwareGrid { display: grid; grid-template-columns: minmax(260px, 1fr) minmax(160px, 220px) minmax(220px, 280px); @@ -1074,6 +1103,8 @@ input[type="text"] { width: 100%; } .fileItem { align-items: flex-start; flex-direction: column; } .firmwareGrid { grid-template-columns: 1fr; } + .autoGrid { grid-template-columns: 1fr; } + .autoGrid .wide { grid-column: auto; } .firmwareLayout, .firmwareTargetGrid, .firmwareActionsPanel { @@ -1164,6 +1195,7 @@
+ @@ -1250,6 +1282,160 @@
+ +