mirror of
https://github.com/TaterTotterson/microWakeWord-Trainer-Nvidia-Docker.git
synced 2026-08-12 07:55:33 -06:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d341d0617 | ||
|
|
a1b22200e0 | ||
|
|
89260f1f14 | ||
|
|
0140dfb56f | ||
|
|
1fc7d80bae | ||
|
|
31a6388da4 | ||
|
|
85c2d6334b | ||
|
|
5f6f108c85 |
70
.github/workflows/docker-publish.yml
vendored
70
.github/workflows/docker-publish.yml
vendored
@@ -7,7 +7,7 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
@@ -41,6 +41,8 @@ jobs:
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
flavor: |
|
||||
latest=false
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=ref,event=tag
|
||||
@@ -56,3 +58,69 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha,scope=mww-trainer-nvidia-docker
|
||||
cache-to: type=gha,mode=max,scope=mww-trainer-nvidia-docker
|
||||
|
||||
- name: Docker metadata (Blackwell)
|
||||
id: meta-blackwell
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
flavor: |
|
||||
latest=false
|
||||
tags: |
|
||||
type=raw,value=blackwell
|
||||
type=ref,event=tag,suffix=-blackwell
|
||||
|
||||
- name: Build and push Blackwell image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: dockerfile.blackwell
|
||||
platforms: linux/amd64
|
||||
push: true
|
||||
tags: ${{ steps.meta-blackwell.outputs.tags }}
|
||||
labels: ${{ steps.meta-blackwell.outputs.labels }}
|
||||
cache-from: type=gha,scope=mww-trainer-nvidia-docker-blackwell
|
||||
cache-to: type=gha,mode=max,scope=mww-trainer-nvidia-docker-blackwell
|
||||
|
||||
- name: Create release notes
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG_NAME: ${{ github.ref_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
title="microWakeWord Nvidia Trainer ${TAG_NAME}"
|
||||
generated_notes="$(mktemp)"
|
||||
release_notes="$(mktemp)"
|
||||
|
||||
gh api "repos/${REPO}/releases/generate-notes" \
|
||||
-f tag_name="${TAG_NAME}" \
|
||||
-f target_commitish="${GITHUB_SHA}" \
|
||||
--jq '.body' > "${generated_notes}"
|
||||
|
||||
{
|
||||
echo "## Docker Images"
|
||||
echo
|
||||
echo "- \`ghcr.io/tatertotterson/microwakeword:${TAG_NAME}\`"
|
||||
echo "- \`ghcr.io/tatertotterson/microwakeword:latest\`"
|
||||
echo "- \`ghcr.io/tatertotterson/microwakeword:${TAG_NAME}-blackwell\`"
|
||||
echo "- \`ghcr.io/tatertotterson/microwakeword:blackwell\`"
|
||||
echo
|
||||
cat "${generated_notes}"
|
||||
} > "${release_notes}"
|
||||
|
||||
if gh release view "${TAG_NAME}" >/dev/null 2>&1; then
|
||||
gh release edit "${TAG_NAME}" \
|
||||
--title "${title}" \
|
||||
--notes-file "${release_notes}" \
|
||||
--latest \
|
||||
--verify-tag
|
||||
else
|
||||
gh release create "${TAG_NAME}" \
|
||||
--title "${title}" \
|
||||
--notes-file "${release_notes}" \
|
||||
--latest \
|
||||
--verify-tag
|
||||
fi
|
||||
|
||||
112
README.md
112
README.md
@@ -7,7 +7,7 @@
|
||||
<a href="https://taterassistant.com">taterassistant.com</a>
|
||||
</h3>
|
||||
|
||||
Train custom microWakeWord models in Docker with NVIDIA/CUDA acceleration, generated Piper samples, device-captured samples, reviewed false-wake negatives, live training logs, and prebuilt Tater firmware flashing.
|
||||
Train custom microWakeWord models in Docker with NVIDIA/CUDA acceleration, generated Piper samples, device-captured samples, reviewed false-wake negatives, live training logs, and local wake-word links for Tater Native satellites.
|
||||
|
||||
Real samples come from device-captured wake audio, close misses, or manual uploads. Every saved sample is normalized to `16 kHz / mono / 16-bit PCM WAV` before training.
|
||||
|
||||
@@ -22,9 +22,22 @@ docker pull ghcr.io/tatertotterson/microwakeword:latest
|
||||
Tagged releases also publish matching immutable image tags:
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/tatertotterson/microwakeword:v5
|
||||
docker pull ghcr.io/tatertotterson/microwakeword:v11
|
||||
```
|
||||
|
||||
RTX 50-series / Blackwell GPUs use a separate image with CUDA 12.8 and a
|
||||
Python 3.13 TensorFlow build for `sm_120`:
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/tatertotterson/microwakeword:blackwell
|
||||
docker pull ghcr.io/tatertotterson/microwakeword:v11-blackwell
|
||||
```
|
||||
|
||||
Use the Blackwell image only for RTX 50-series cards. It includes the
|
||||
community-built TensorFlow wheel from
|
||||
[chivitiH/tensorflow-blackwell-python313](https://github.com/chivitiH/tensorflow-blackwell-python313),
|
||||
which is unofficial and licensed CC BY-NC 4.0.
|
||||
|
||||
---
|
||||
|
||||
## Run The Container
|
||||
@@ -38,16 +51,19 @@ docker run -d \
|
||||
ghcr.io/tatertotterson/microwakeword:latest
|
||||
```
|
||||
|
||||
Use a version tag such as `ghcr.io/tatertotterson/microwakeword:v5` when you want to pin a known release instead of tracking `latest`.
|
||||
Use a version tag such as `ghcr.io/tatertotterson/microwakeword:v11` when you want to pin a known release instead of tracking `latest`.
|
||||
For RTX 50-series cards, use `ghcr.io/tatertotterson/microwakeword:blackwell`
|
||||
or a pinned tag such as `ghcr.io/tatertotterson/microwakeword:v11-blackwell`
|
||||
in the same `docker run` command.
|
||||
|
||||
The flags:
|
||||
|
||||
- `--gpus all` enables GPU acceleration.
|
||||
- `--network host` lets the container receive mDNS/zeroconf traffic for ESPHome auto-detect.
|
||||
- `--network host` exposes the trainer server directly so satellites can send captured audio and load trained wake-word files.
|
||||
- `-e REC_PORT=8789` sets the trainer web UI and captured-audio port. Change this value if `8789` is already in use.
|
||||
- `-v $(pwd):/data` persists models, downloaded voices, datasets, samples, and firmware caches.
|
||||
- `-v $(pwd):/data` persists models, downloaded voices, datasets, samples, and generated wake-word artifacts.
|
||||
|
||||
Host networking is recommended for the Firmware tab's mDNS device discovery. Manual IP flashing and captured-audio uploads can still work without host networking if the trainer port is reachable, but auto-detect may not see devices from Docker bridge networking.
|
||||
If you do not use host networking, publish the trainer port and make sure satellites can reach it from your LAN.
|
||||
|
||||
Open:
|
||||
|
||||
@@ -55,31 +71,37 @@ Open:
|
||||
http://localhost:8789
|
||||
```
|
||||
|
||||
If you change `REC_PORT`, open that port instead and use the same port in the ESPHome `Trainer App URL`.
|
||||
If you change `REC_PORT`, open that port instead and use the same port in the satellite `Trainer App URL`.
|
||||
|
||||
---
|
||||
|
||||
## What The UI Does
|
||||
|
||||
- `Trainer` starts a wake-word session, shows positive/negative sample counts, and launches training.
|
||||
- `Captured Audio` reviews clips sent by ESPHome sats, including wake hits, close misses, and false wakes.
|
||||
- `Captured Audio` reviews clips sent by Tater Native or ESPHome sats, including wake hits, close misses, and false wakes.
|
||||
- `Samples` plays, removes, clears, and manually imports personal or negative samples.
|
||||
- `Firmware` pulls verified prebuilt Tater firmware images from GitHub and flashes supported satellites over OTA.
|
||||
- Popup consoles show colorized training and firmware logs while long-running jobs are active.
|
||||
- `Wake Words` lists locally trained JSON/model links for live wake-word switching in Tater.
|
||||
- Popup consoles show colorized training logs while long-running jobs are active.
|
||||
|
||||
---
|
||||
|
||||
## Captured Audio Workflow
|
||||
|
||||
To collect samples from a sat, flash it with the Tater firmware from [TaterTotterson/microWakeWords](https://github.com/TaterTotterson/microWakeWords). The `Firmware` tab can pull verified prebuilt OTA images from that repo for fast firmware updates.
|
||||
To collect samples from a sat, point its trainer feedback setting at this app. Tater Native satellites use the native settings popup in Tater. Older ESPHome satellites can still use their device entities.
|
||||
|
||||
After flashing, the device exposes ESPHome entities for capture setup:
|
||||
For Tater Native satellites, enable trainer feedback in Tater:
|
||||
|
||||
- `Send Good Wakes To Trainer` toggles upload of confirmed wake-word triggers.
|
||||
- `Send Close Misses To Trainer` toggles upload of near misses.
|
||||
- `Trainer App URL` sets the trainer address, for example `http://trainer.local:8789` or `http://<trainer-ip>:8789`.
|
||||
|
||||
For older ESPHome firmware, the equivalent capture setup is exposed as device entities:
|
||||
|
||||
- `Capture Wake Audio` toggles upload of wake-word triggers.
|
||||
- `Capture Close Misses` toggles upload of near misses.
|
||||
- `Trainer App URL` sets the trainer address, for example `http://<trainer-ip>:8789`.
|
||||
|
||||
ESPHome devices can send raw captured audio to:
|
||||
Satellites send raw captured audio to:
|
||||
|
||||
```text
|
||||
/api/upload_captured_audio_raw
|
||||
@@ -153,6 +175,8 @@ Personal samples are optional. Training can run with zero personal samples after
|
||||
|
||||
Reviewed negative samples are converted into `/data/work/reviewed_negative_features/` and inserted into the training YAML as a hard-negative feature set when present.
|
||||
|
||||
On RTX 50-series / Blackwell GPUs, the Blackwell Docker image keeps sample generation and augmentation in the normal Python 3.12 trainer environment, then runs only the TensorFlow training/export stage in `/data/.venv-blackwell` with Python 3.13 and the Blackwell-native TensorFlow wheel.
|
||||
|
||||
---
|
||||
|
||||
## Language Support
|
||||
@@ -183,20 +207,16 @@ After those assets are prepared, later runs reuse the local copies unless the mo
|
||||
|
||||
---
|
||||
|
||||
## Firmware Flashing
|
||||
## Trained Wake Words
|
||||
|
||||
The `Firmware` tab flashes prebuilt Tater firmware for supported ESPHome satellites.
|
||||
The `Wake Words` tab lists locally trained wake-word packages from `/data/trained_wake_words/`.
|
||||
|
||||
- Downloads the latest prebuilt firmware manifest plus OTA and USB factory images from `TaterTotterson/microWakeWords`.
|
||||
- Verifies downloaded images by size and SHA before upload.
|
||||
- Auto-detects ESPHome devices with mDNS when the container is running with host networking.
|
||||
- Allows manual IP or hostname entry if discovery does not find the device.
|
||||
- Saves the selected OTA target for each firmware family.
|
||||
- Flashes the prebuilt factory image over Browser USB for first installs or recovery when opened in Chrome or Edge.
|
||||
- Lists locally trained wake words from `/data/trained_wake_words/` for live model switching.
|
||||
- Streams download, verification, and OTA upload progress in a colorized firmware console.
|
||||
- Copy the JSON URL into the Tater Native satellite settings to switch wake words live.
|
||||
- Open the JSON or model links directly for quick inspection.
|
||||
- The JSON includes the matching model path plus Tater tuning metadata.
|
||||
- No firmware flashing happens from this trainer app anymore.
|
||||
|
||||
You usually only flash for firmware updates. New satellites, or devices older than Tater firmware `3.0.3`, need one USB flash first before OTA updates and live wake-word switching are available.
|
||||
Use the main Tater app for satellite firmware updates and USB flashing.
|
||||
|
||||
---
|
||||
|
||||
@@ -209,14 +229,50 @@ Successful runs produce timestamped training output folders such as:
|
||||
/data/output/<timestamp>-<wake_word>-<samples>-<steps>/<wake_word>.json
|
||||
```
|
||||
|
||||
The trainer also syncs firmware-ready artifacts into:
|
||||
The trainer also syncs Tater-ready wake-word artifacts into:
|
||||
|
||||
```text
|
||||
/data/trained_wake_words/<wake_word>.tflite
|
||||
/data/trained_wake_words/<wake_word>.json
|
||||
```
|
||||
|
||||
The firmware tab uses `/data/trained_wake_words/` to populate the wake-word dropdown.
|
||||
The `Wake Words` tab uses `/data/trained_wake_words/` to populate the local wake-word links.
|
||||
|
||||
The JSON keeps the standard microWakeWord fields for compatibility:
|
||||
|
||||
```json
|
||||
{
|
||||
"micro": {
|
||||
"probability_cutoff": 0.97,
|
||||
"sliding_window_size": 5
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
It also includes Tater Native metadata used by newer satellites and the Tater settings UI:
|
||||
|
||||
```json
|
||||
{
|
||||
"model_format": "tflite_stream_state_internal_quant",
|
||||
"quantization": "int8",
|
||||
"sample_rate": 16000,
|
||||
"tater_native": {
|
||||
"format_version": 1,
|
||||
"wake_threshold": 0.97,
|
||||
"wake_sliding_window": 5,
|
||||
"close_miss_threshold": 0.78,
|
||||
"frontend": {
|
||||
"name": "tflm_microfrontend",
|
||||
"sample_rate": 16000,
|
||||
"feature_duration_ms": 30,
|
||||
"feature_step_ms": 10,
|
||||
"feature_size": 40
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Calibration metrics are included under `calibration` so false accepts/hour and recall can be surfaced in the UI.
|
||||
|
||||
---
|
||||
|
||||
@@ -233,7 +289,6 @@ That removes:
|
||||
- cached datasets
|
||||
- training environments
|
||||
- trained models
|
||||
- downloaded firmware images
|
||||
|
||||
---
|
||||
|
||||
@@ -243,7 +298,7 @@ That removes:
|
||||
- Negative samples are optional but useful for reducing false wakes.
|
||||
- The UI server is `trainer_server.py`.
|
||||
- The launcher is `run.sh`.
|
||||
- Firmware capture settings live on the ESPHome device and can be toggled from the device entities after flashing.
|
||||
- Trainer capture settings live in Tater for Tater Native satellites, and on device entities for older ESPHome satellites.
|
||||
|
||||
---
|
||||
|
||||
@@ -253,3 +308,4 @@ Built on top of:
|
||||
|
||||
- [microWakeWord](https://github.com/kahrendt/microWakeWord)
|
||||
- [piper-sample-generator](https://github.com/rhasspy/piper-sample-generator)
|
||||
- [tensorflow-blackwell-python313](https://github.com/chivitiH/tensorflow-blackwell-python313) for the optional RTX 50-series / Blackwell image
|
||||
|
||||
@@ -18,14 +18,14 @@ from microwakeword.data import FeatureHandler
|
||||
from microwakeword.inference import Model
|
||||
|
||||
|
||||
DEFAULT_WINDOW_SIZES = [3, 4, 5, 6, 7]
|
||||
DEFAULT_TARGET_FAPH = float(os.environ.get("MWW_CALIBRATION_TARGET_FAPH", "1.0"))
|
||||
DEFAULT_WINDOW_SIZES = [4, 5, 6, 7]
|
||||
DEFAULT_TARGET_FAPH = float(os.environ.get("MWW_CALIBRATION_TARGET_FAPH", "0.25"))
|
||||
DEFAULT_COOLDOWN_SLICES = int(os.environ.get("MWW_CALIBRATION_COOLDOWN_SLICES", "25"))
|
||||
DEFAULT_POSITIVE_SKIP_SLICES = int(
|
||||
os.environ.get("MWW_CALIBRATION_POSITIVE_SKIP_SLICES", "25")
|
||||
)
|
||||
DEFAULT_CUTOFF_STEP = float(os.environ.get("MWW_CALIBRATION_CUTOFF_STEP", "0.01"))
|
||||
DEFAULT_CUTOFF_MIN = float(os.environ.get("MWW_CALIBRATION_CUTOFF_MIN", "0.00"))
|
||||
DEFAULT_CUTOFF_MIN = float(os.environ.get("MWW_CALIBRATION_CUTOFF_MIN", "0.85"))
|
||||
DEFAULT_CUTOFF_MAX = float(os.environ.get("MWW_CALIBRATION_CUTOFF_MAX", "1.00"))
|
||||
|
||||
|
||||
|
||||
112
cli/setup_blackwell_venv
Executable file
112
cli/setup_blackwell_venv
Executable file
@@ -0,0 +1,112 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
PROGDIR="$(dirname "$(realpath "$0")")"
|
||||
ROOTDIR="$(dirname "${PROGDIR}")"
|
||||
|
||||
KNOWN_ARGS=( data-dir force python )
|
||||
source "${PROGDIR}/shell.functions"
|
||||
|
||||
if [ ${#UNKNOWN_ARGS[@]} -gt 0 ] ; then
|
||||
echo "Unknown argument(s): ${UNKNOWN_ARGS[*]}" >&2
|
||||
HELP=true
|
||||
fi
|
||||
|
||||
if [ "${HELP}" == "true" ] ; then
|
||||
cat <<EOF >&2
|
||||
Usage: setup_blackwell_venv [ --data-dir=/data ] [ --force ] [ --python=python3.13 ]
|
||||
|
||||
Creates /data/.venv-blackwell for RTX 50 / Blackwell TensorFlow training.
|
||||
Sample generation and augmentation continue to use /data/.venv.
|
||||
|
||||
Environment overrides:
|
||||
MWW_BLACKWELL_TF_WHEEL_URL: TensorFlow Blackwell wheel URL.
|
||||
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[ -n "${DATA_DIR}" ] && DATA_DIR="$(realpath "${DATA_DIR}")"
|
||||
[ -d "${DATA_DIR}" ] || {
|
||||
echo "Data directory '${DATA_DIR}' doesn't exist." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
PYTHON="${PYTHON:-python3.13}"
|
||||
VENV="${DATA_DIR}/.venv-blackwell"
|
||||
MARKER="${VENV}/.mww-blackwell-venv"
|
||||
TF_WHEEL_URL="${MWW_BLACKWELL_TF_WHEEL_URL:-https://github.com/chivitiH/tensorflow-blackwell-python313/releases/download/v2.22.0-selfbuilt/tensorflow-2.22.0.dev0+selfbuilt-cp313-cp313-linux_x86_64.whl}"
|
||||
|
||||
if ! command -v "${PYTHON}" >/dev/null 2>&1 ; then
|
||||
echo "Python 3.13 is required for the Blackwell TensorFlow wheel. Missing: ${PYTHON}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${FORCE:-false}" != "true" ] && [ -x "${VENV}/bin/python" ] && [ -f "${MARKER}" ] ; then
|
||||
echo " Blackwell TensorFlow venv found (skipping setup_blackwell_venv)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "===== Setting up Blackwell TensorFlow environment ${VENV} ====="
|
||||
rm -rf "${VENV}" || :
|
||||
"${PYTHON}" -m venv --upgrade-deps "${VENV}"
|
||||
source "${VENV}/bin/activate"
|
||||
|
||||
export PIP_PROGRESS_BAR=off
|
||||
export PIP_NO_COLOR=1
|
||||
export PIP_QUIET=0
|
||||
|
||||
pip_install() {
|
||||
if $VERBOSE ; then
|
||||
pip install "$@" || return 1
|
||||
else
|
||||
{ pip install "$@" || return 1 ; } | stdbuf -i0 -o0 tr -d '[:print:]' | stdbuf -i0 -o0 tr '\n' '.'
|
||||
fi
|
||||
echo
|
||||
}
|
||||
|
||||
echo " ===== Installing Blackwell TensorFlow wheel ====="
|
||||
pip_install --upgrade pip setuptools wheel
|
||||
pip_install "${TF_WHEEL_URL}"
|
||||
|
||||
echo " ===== Installing microWakeWord training dependencies ====="
|
||||
pip_install \
|
||||
audiomentations \
|
||||
audio_metadata \
|
||||
datasets \
|
||||
mmap_ninja \
|
||||
pymicro-features \
|
||||
pyyaml \
|
||||
webrtcvad-wheels \
|
||||
ai-edge-litert \
|
||||
numpy-minmax \
|
||||
numpy-rms \
|
||||
absl-py \
|
||||
"numpy==2.3.5"
|
||||
|
||||
echo " ===== Checking microwakeword ====="
|
||||
MWW="${DATA_DIR}/tools/microWakeWord"
|
||||
if [ ! -d "${MWW}" ] || [ -n "$(git -C "${MWW}" status --porcelain 2>/dev/null || true)" ] ; then
|
||||
rm -rf "${MWW}" || :
|
||||
mkdir -p "${DATA_DIR}/tools"
|
||||
echo " Cloning micro-wake-word to ${DATA_DIR}/tools"
|
||||
git clone https://github.com/TaterTotterson/micro-wake-word "${MWW}" &>/dev/null
|
||||
fi
|
||||
echo " Installing microwakeword into Blackwell venv"
|
||||
pip_install --no-deps -e "${MWW}"
|
||||
|
||||
echo " ===== Testing Blackwell TensorFlow environment ====="
|
||||
"${VENV}/bin/python" - <<'PY'
|
||||
import tensorflow as tf
|
||||
from ai_edge_litert.interpreter import Interpreter
|
||||
from microwakeword.data import FeatureHandler
|
||||
from microwakeword.inference import Model
|
||||
|
||||
print("TensorFlow:", tf.__version__)
|
||||
print("CUDA build:", tf.test.is_built_with_cuda())
|
||||
print("GPU:", tf.config.list_physical_devices("GPU"))
|
||||
print("microWakeWord Blackwell imports available")
|
||||
PY
|
||||
|
||||
touch "${MARKER}"
|
||||
echo "Blackwell TensorFlow environment ready: ${VENV}"
|
||||
@@ -25,9 +25,9 @@ fi
|
||||
mkdir -p "${DATA_DIR}/training_datasets/downloads" || :
|
||||
cd "${DATA_DIR}/training_datasets"
|
||||
|
||||
AUDIO_URL="https://mcdermottlab.mit.edu/Reverb/IRMAudio/Audio.zip"
|
||||
AUDIO_ZIPFILE="MIT_RIR_Audio.zip"
|
||||
AUDIO_ZIP="./downloads/${AUDIO_ZIPFILE}"
|
||||
HF_RIR_REPO_ID="TaterTotterson/MIT_environmental_impulse_responses"
|
||||
HF_RIR_API_URL="https://huggingface.co/api/datasets/${HF_RIR_REPO_ID}"
|
||||
HF_RIR_SOURCE_KEY="hf_mit_environmental_impulse_responses"
|
||||
AUDIO_DIR="./mit_rirs"
|
||||
mkdir -p "${AUDIO_DIR}" || :
|
||||
AUDIO16K_DIR="./mit_rirs_16k"
|
||||
@@ -35,10 +35,92 @@ mkdir -p "${AUDIO16K_DIR}" || :
|
||||
AUDIO_FILECOUNT="./downloads/mit_rir_filecount"
|
||||
AUDIO_IN_GLOB="*.wav"
|
||||
|
||||
declare -A filecounts=( [${AUDIO_ZIPFILE}]=0 )
|
||||
declare -A filecounts=( [${HF_RIR_SOURCE_KEY}]=0 )
|
||||
get_filecounts filecounts "${AUDIO_FILECOUNT}"
|
||||
|
||||
echo "===== Checking MIT_RIR ====="
|
||||
echo "===== Checking MIT environmental RIRs ====="
|
||||
|
||||
download_hf_mit_rirs() {
|
||||
source ${DATA_DIR}/.venv/bin/activate
|
||||
python - "${HF_RIR_REPO_ID}" "${HF_RIR_API_URL}" "${AUDIO_DIR}" <<-'EOF'
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
repo_id = sys.argv[1]
|
||||
api_url = sys.argv[2]
|
||||
audio_dir = Path(sys.argv[3])
|
||||
audio_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
request = urllib.request.Request(api_url, headers={"User-Agent": "WakeWordTrainer/1.0"})
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
metadata = json.loads(response.read().decode("utf-8"))
|
||||
|
||||
files = sorted(
|
||||
sibling.get("rfilename", "")
|
||||
for sibling in metadata.get("siblings", [])
|
||||
if str(sibling.get("rfilename", "")).startswith("16khz/")
|
||||
and str(sibling.get("rfilename", "")).lower().endswith(".wav")
|
||||
)
|
||||
if not files:
|
||||
raise SystemExit("Hugging Face MIT RIR dataset did not list any 16khz WAV files")
|
||||
|
||||
print(f" Found {len(files)} MIT environmental RIR files on Hugging Face mirror", flush=True)
|
||||
downloaded = 0
|
||||
skipped = 0
|
||||
|
||||
def download_file(url: str, target: Path, rel: str):
|
||||
tmp = target.with_suffix(target.suffix + ".incomplete")
|
||||
for attempt in range(1, 4):
|
||||
try:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
with urllib.request.urlopen(url, timeout=30) as response:
|
||||
with tmp.open("wb") as out:
|
||||
while True:
|
||||
chunk = response.read(1024 * 64)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
if not tmp.exists() or tmp.stat().st_size == 0:
|
||||
raise RuntimeError("empty download")
|
||||
tmp.replace(target)
|
||||
return
|
||||
except Exception as exc:
|
||||
if tmp.exists():
|
||||
tmp.unlink()
|
||||
if attempt == 3:
|
||||
raise RuntimeError(f"download failed for {rel}: {exc}") from exc
|
||||
print(f" Retry {attempt}/2 for {rel}: {exc}", flush=True)
|
||||
time.sleep(2 * attempt)
|
||||
|
||||
total = len(files)
|
||||
for idx, rel in enumerate(files, start=1):
|
||||
target = audio_dir / rel
|
||||
if target.exists() and target.stat().st_size > 0:
|
||||
skipped += 1
|
||||
if idx == 1 or idx % 25 == 0 or idx == total:
|
||||
print(f" MIT RIR download progress: {idx}/{total} files ({downloaded} downloaded, {skipped} reused)", flush=True)
|
||||
continue
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
encoded = urllib.parse.quote(rel, safe="/")
|
||||
url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{encoded}"
|
||||
if idx == 1 or idx % 25 == 0 or idx == total:
|
||||
print(f" Downloading MIT RIR {idx}/{total}: {rel}", flush=True)
|
||||
download_file(url, target, rel)
|
||||
if not target.exists() or target.stat().st_size == 0:
|
||||
raise SystemExit(f"download failed for {rel}")
|
||||
downloaded += 1
|
||||
if idx == 1 or idx % 25 == 0 or idx == total:
|
||||
print(f" MIT RIR download progress: {idx}/{total} files ({downloaded} downloaded, {skipped} reused)", flush=True)
|
||||
|
||||
print(f" Hugging Face MIT environmental RIR download complete ({downloaded} downloaded, {skipped} reused)", flush=True)
|
||||
print(f" MIT environmental RIR files available: {len(files)}", flush=True)
|
||||
EOF
|
||||
}
|
||||
|
||||
converter() {
|
||||
source ${DATA_DIR}/.venv/bin/activate
|
||||
@@ -58,9 +140,9 @@ rir_out = Path(sys.argv[2])
|
||||
|
||||
waves = list(rir_in.rglob("*.wav"))
|
||||
try:
|
||||
print(" MIT RIR normalizing to 16k…")
|
||||
print(" MIT environmental RIR normalizing to 16k…")
|
||||
# Normalize to 16k mono
|
||||
for p in tqdm(waves, desc=" MIT_RIR (resample 16k mono)"):
|
||||
for p in tqdm(waves, desc=" MIT environmental RIR (resample 16k mono)"):
|
||||
outfile = Path(rir_out / p.name)
|
||||
if outfile.exists():
|
||||
continue
|
||||
@@ -70,14 +152,14 @@ try:
|
||||
if sr != 16000:
|
||||
a, _ = librosa.load(p, sr=16000, mono=True)
|
||||
write_wav(outfile, a, 16000)
|
||||
print(" MIT RIR normalization complete")
|
||||
print(" MIT environmental RIR normalization complete")
|
||||
except Exception as e2:
|
||||
print(f" MIT RIR fallback failed: {e2}")
|
||||
print(f" MIT environmental RIR preparation failed: {e2}")
|
||||
raise
|
||||
EOF
|
||||
}
|
||||
|
||||
expected_filecount=${filecounts[${AUDIO_ZIPFILE}]}
|
||||
expected_filecount=${filecounts[${HF_RIR_SOURCE_KEY}]}
|
||||
actual_filecount=$(find "${AUDIO16K_DIR}" -name '*.wav' 2>/dev/null | wc -l) || :
|
||||
write_filecount=false
|
||||
|
||||
@@ -85,24 +167,16 @@ if [ "${actual_filecount}" -ne 0 ] && [ "${actual_filecount}" -eq "${expected_fi
|
||||
echo " Existing ${AUDIO16K_DIR} valid"
|
||||
else
|
||||
actual_filecount=$(find "${AUDIO_DIR}" -name "${AUDIO_IN_GLOB}" 2>/dev/null | wc -l) || :
|
||||
if [ "${actual_filecount}" -eq 0 ] || [ "${actual_filecount}" -ne "${expected_filecount}" ] ; then
|
||||
if [ ! -f "${AUDIO_ZIP}" ] ; then
|
||||
echo " Downloading ${AUDIO_ZIPFILE}"
|
||||
curl -sfL "${AUDIO_URL}" -o "${AUDIO_ZIP}"
|
||||
fi
|
||||
|
||||
if [ "${actual_filecount}" -eq 0 ] || [ "${expected_filecount}" -eq 0 ] || [ "${actual_filecount}" -ne "${expected_filecount}" ] ; then
|
||||
rm -rf "${AUDIO_DIR}" || :
|
||||
echo " Unzipping ${AUDIO_ZIPFILE}"
|
||||
unzip -u -q -d "${AUDIO_DIR}" "${AUDIO_ZIP}"
|
||||
fi
|
||||
if "${CLEANUP_ARCHIVES}" && [ -f "${AUDIO_ZIP}" ] ; then
|
||||
echo " Cleaning up ${AUDIO_ZIPFILE}"
|
||||
rm -rf "${AUDIO_ZIP}"
|
||||
mkdir -p "${AUDIO_DIR}" || :
|
||||
echo " Downloading MIT environmental impulse responses from Hugging Face mirror"
|
||||
download_hf_mit_rirs
|
||||
fi
|
||||
|
||||
converter
|
||||
actual_filecount=$(find "${AUDIO16K_DIR}" -name "*.wav" 2>/dev/null | wc -l) || :
|
||||
filecounts[${AUDIO_ZIPFILE}]="${actual_filecount}"
|
||||
filecounts[${HF_RIR_SOURCE_KEY}]="${actual_filecount}"
|
||||
write_filecount=true
|
||||
fi
|
||||
|
||||
@@ -110,15 +184,10 @@ if ${write_filecount} ; then
|
||||
write_filecounts filecounts "${AUDIO_FILECOUNT}"
|
||||
fi
|
||||
|
||||
if "${CLEANUP_ARCHIVES}" && [ -f "${AUDIO_ZIP}" ] ; then
|
||||
echo " Cleaning up ${AUDIO_ZIPFILE}"
|
||||
rm -rf "${AUDIO_ZIP}"
|
||||
fi
|
||||
|
||||
if "${CLEANUP_INTERMEDIATE_FILES}" && [ -d "${AUDIO_DIR}" ]; then
|
||||
echo " Cleaning up ${AUDIO_DIR}"
|
||||
rm -rf "${AUDIO_DIR}"
|
||||
fi
|
||||
|
||||
echo " MIT_RIR complete"
|
||||
echo " MIT environmental RIRs complete"
|
||||
exit 0
|
||||
|
||||
@@ -84,6 +84,21 @@ if [ "${IS_BLACKWELL}" = "true" ]; then
|
||||
echo "ℹ️ Using GPU compatibility retries; CPU fallback is ${ALLOW_CPU_FALLBACK} (override with MWW_ALLOW_CPU_FALLBACK=true|false)."
|
||||
fi
|
||||
|
||||
BLACKWELL_TF_MODE="${MWW_BLACKWELL_TF:-auto}"
|
||||
BLACKWELL_TF_REQUIRED="false"
|
||||
BLACKWELL_TF_ACTIVE="false"
|
||||
case "${BLACKWELL_TF_MODE,,}" in
|
||||
1|true|yes|on|required)
|
||||
BLACKWELL_TF_REQUIRED="true"
|
||||
;;
|
||||
0|false|no|off|disabled)
|
||||
BLACKWELL_TF_MODE="disabled"
|
||||
;;
|
||||
*)
|
||||
BLACKWELL_TF_MODE="auto"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Enable driver-side PTX JIT fallback when ptxas/nvlink are unavailable.
|
||||
if [ -z "${XLA_FLAGS:-}" ]; then
|
||||
export XLA_FLAGS="--xla_gpu_unsafe_fallback_to_driver_on_ptxas_not_found"
|
||||
@@ -238,6 +253,32 @@ fi
|
||||
echo " Wrote training_parameters.yaml"
|
||||
rm -rf "${WORK_DIR}/trained_models/wakeword"
|
||||
|
||||
if [ "${IS_BLACKWELL}" = "true" ] && [ "${BLACKWELL_TF_MODE}" != "disabled" ]; then
|
||||
BLACKWELL_SETUP="${PROGDIR}/setup_blackwell_venv"
|
||||
BLACKWELL_PYTHON="${DATA_DIR}/.venv-blackwell/bin/python"
|
||||
|
||||
if [ -x "${BLACKWELL_SETUP}" ] && command -v python3.13 >/dev/null 2>&1; then
|
||||
echo "↪️ Preparing Blackwell-native TensorFlow training environment."
|
||||
if "${BLACKWELL_SETUP}" --data-dir="${DATA_DIR}"; then
|
||||
PYTHON_BIN="${BLACKWELL_PYTHON}"
|
||||
BLACKWELL_TF_ACTIVE="true"
|
||||
echo "✅ Blackwell TensorFlow training enabled: ${PYTHON_BIN}"
|
||||
else
|
||||
if [ "${BLACKWELL_TF_REQUIRED}" = "true" ]; then
|
||||
echo "❌ Blackwell TensorFlow setup failed and MWW_BLACKWELL_TF is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "⚠️ Blackwell TensorFlow setup failed; continuing with compatibility retries."
|
||||
fi
|
||||
else
|
||||
if [ "${BLACKWELL_TF_REQUIRED}" = "true" ]; then
|
||||
echo "❌ Blackwell TensorFlow was required, but python3.13/setup_blackwell_venv is unavailable." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "ℹ️ Blackwell TensorFlow image support not available; continuing with compatibility retries."
|
||||
fi
|
||||
fi
|
||||
|
||||
wake_word_filename="$(
|
||||
echo "${WAKE_WORD}" \
|
||||
| tr '[:upper:]' '[:lower:]' \
|
||||
@@ -261,11 +302,11 @@ TRAIN_ARGS=(
|
||||
--test_tflite_streaming_quantized 1
|
||||
--use_weights best_weights
|
||||
mixednet
|
||||
--pointwise_filters "64,64,64,64"
|
||||
--pointwise_filters "128,128,128,128"
|
||||
--repeat_in_block "1,1,1,1"
|
||||
--mixconv_kernel_sizes "[5], [7,11], [9,15], [23]"
|
||||
--residual_connection "0,0,0,0"
|
||||
--first_conv_filters 32
|
||||
--first_conv_filters 64
|
||||
--first_conv_kernel_size 5
|
||||
--stride 2
|
||||
)
|
||||
@@ -345,6 +386,7 @@ fi
|
||||
TRAINING_DONE="false"
|
||||
|
||||
echo "🏋️ Starting model training and TFLite export (this is the longest stage)…"
|
||||
echo "🧠 Model quality: high_accuracy_plus"
|
||||
if run_attempt "Attempt 1/3: GPU training (default runtime profile)" ; then
|
||||
echo "✅ Training complete (GPU path)."
|
||||
TRAINING_DONE="true"
|
||||
@@ -454,8 +496,10 @@ from pathlib import Path
|
||||
json_path = Path(os.environ["JSON_PATH"])
|
||||
calibration_path = Path(os.environ.get("CALIBRATION_PATH", ""))
|
||||
language = (os.environ.get("LANGUAGE", "en") or "en").strip().lower()
|
||||
probability_cutoff = 0.97
|
||||
sliding_window_size = 5
|
||||
probability_cutoff = 0.85
|
||||
sliding_window_size = 4
|
||||
strict_min_close_miss_threshold = 0.68
|
||||
calibration = {}
|
||||
|
||||
if calibration_path.exists():
|
||||
try:
|
||||
@@ -469,21 +513,63 @@ if calibration_path.exists():
|
||||
except Exception as exc:
|
||||
print(f"⚠️ Failed to read detector calibration ({exc}); using defaults.")
|
||||
|
||||
probability_cutoff = round(probability_cutoff, 3)
|
||||
sliding_window_size = max(1, min(10, int(sliding_window_size)))
|
||||
selected_metrics = calibration.get("selected_metrics") if isinstance(calibration.get("selected_metrics"), dict) else {}
|
||||
evaluation = calibration.get("evaluation") if isinstance(calibration.get("evaluation"), dict) else {}
|
||||
close_miss_threshold = max(
|
||||
0.01,
|
||||
min(0.99, round(max(strict_min_close_miss_threshold, probability_cutoff - 0.17), 3)),
|
||||
)
|
||||
|
||||
meta = {
|
||||
"type": "micro",
|
||||
"wake_word": os.environ["WAKE_WORD_TITLE"],
|
||||
"label": os.environ["WAKE_WORD_TITLE"].replace("_", " ").title(),
|
||||
"author": "Tater Totterson",
|
||||
"website": "https://github.com/TaterTotterson/microWakeWord-Trainer-Nvidia-Docker.git",
|
||||
"model": os.environ["TFLITE_FILENAME"],
|
||||
"trained_languages": [language],
|
||||
"version": 2,
|
||||
"model_format": "tflite_stream_state_internal_quant",
|
||||
"quantization": "int8",
|
||||
"sample_rate": 16000,
|
||||
"micro": {
|
||||
"probability_cutoff": round(probability_cutoff, 2),
|
||||
"probability_cutoff": probability_cutoff,
|
||||
"sliding_window_size": sliding_window_size,
|
||||
"feature_step_size": 10,
|
||||
"tensor_arena_size": 30000,
|
||||
"minimum_esphome_version": "2024.7.0",
|
||||
},
|
||||
"tater_native": {
|
||||
"format_version": 1,
|
||||
"wake_threshold": probability_cutoff,
|
||||
"wake_sliding_window": sliding_window_size,
|
||||
"close_miss_threshold": close_miss_threshold,
|
||||
"frontend": {
|
||||
"name": "tflm_microfrontend",
|
||||
"sample_rate": 16000,
|
||||
"feature_duration_ms": 30,
|
||||
"feature_step_ms": 10,
|
||||
"feature_size": 40,
|
||||
"input_feature_frames": 2,
|
||||
"lower_band_limit": 125.0,
|
||||
"upper_band_limit": 7500.0,
|
||||
},
|
||||
"recommended_for": ["tater-native-satellite", "voice-pe"],
|
||||
},
|
||||
"calibration": {
|
||||
"target_false_accepts_per_hour": calibration.get("target_false_accepts_per_hour"),
|
||||
"selected_false_accepts_per_hour_limit": calibration.get("selected_false_accepts_per_hour_limit"),
|
||||
"recall": selected_metrics.get("recall"),
|
||||
"false_accepts_per_hour": selected_metrics.get("false_accepts_per_hour"),
|
||||
"ambient_hours": selected_metrics.get("ambient_hours"),
|
||||
"positive_dataset": evaluation.get("positive_dataset"),
|
||||
"ambient_dataset": evaluation.get("ambient_dataset"),
|
||||
"positive_tracks": evaluation.get("positive_tracks"),
|
||||
"ambient_tracks": evaluation.get("ambient_tracks"),
|
||||
"generated_at": calibration.get("generated_at"),
|
||||
},
|
||||
}
|
||||
json_path.write_text(json.dumps(meta, indent=4) + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
54
dockerfile.blackwell
Normal file
54
dockerfile.blackwell
Normal file
@@ -0,0 +1,54 @@
|
||||
# RTX 50 / Blackwell image
|
||||
FROM nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV CUDA_HOME=/usr/local/cuda
|
||||
ENV PATH=/usr/local/cuda/bin:${PATH}
|
||||
ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:${LD_LIBRARY_PATH}
|
||||
ENV MWW_BLACKWELL_IMAGE=1
|
||||
ENV MWW_BLACKWELL_TF=auto
|
||||
ENV MWW_BLACKWELL_TF_WHEEL_URL=https://github.com/chivitiH/tensorflow-blackwell-python313/releases/download/v2.22.0-selfbuilt/tensorflow-2.22.0.dev0+selfbuilt-cp313-cp313-linux_x86_64.whl
|
||||
|
||||
# System deps. Python 3.12 remains the main trainer/runtime venv, while
|
||||
# Python 3.13 is used only for the Blackwell TensorFlow training step.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
software-properties-common ca-certificates curl git wget unzip patch \
|
||||
ninja-build nano less \
|
||||
&& add-apt-repository -y ppa:deadsnakes/ppa \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
python3.12 python3.12-venv python3.12-dev \
|
||||
python3.13 python3.13-venv python3.13-dev \
|
||||
python3-pip python-is-python3 \
|
||||
&& ldconfig \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& mkdir -p /data
|
||||
|
||||
# Trainer UI port
|
||||
EXPOSE 8789
|
||||
|
||||
# Script root
|
||||
WORKDIR /root/mww-scripts
|
||||
|
||||
# Bash environment
|
||||
COPY --chown=root:root --chmod=0755 .bashrc /root/
|
||||
|
||||
# Root-level entrypoints
|
||||
COPY --chown=root:root --chmod=0755 \
|
||||
train_wake_word \
|
||||
run.sh \
|
||||
trainer_server.py \
|
||||
requirements.txt \
|
||||
/root/mww-scripts/
|
||||
|
||||
# CLI folder
|
||||
COPY --chown=root:root cli/ /root/mww-scripts/cli/
|
||||
|
||||
# Make all CLI scripts executable (avoids "Permission denied")
|
||||
RUN chmod -R a+x /root/mww-scripts/cli
|
||||
|
||||
# Static UI for trainer
|
||||
COPY --chown=root:root --chmod=0644 static/index.html /root/mww-scripts/static/index.html
|
||||
|
||||
# trainer server
|
||||
CMD ["/bin/bash", "-lc", "/root/mww-scripts/run.sh"]
|
||||
2
run.sh
2
run.sh
@@ -30,7 +30,6 @@ install_ui_deps() {
|
||||
"fastapi==${FASTAPI_VERSION}" \
|
||||
"uvicorn[standard]==${UVICORN_VERSION}" \
|
||||
"python-multipart==${PY_MULTIPART_VERSION}" \
|
||||
"zeroconf>=0.132.2" \
|
||||
"silero-vad>=5.0.0" \
|
||||
"numpy>=1.24.0"
|
||||
}
|
||||
@@ -79,7 +78,6 @@ exact = {
|
||||
minimum = {
|
||||
"silero-vad": "5.0.0",
|
||||
"numpy": "1.24.0",
|
||||
"zeroconf": "0.132.2",
|
||||
}
|
||||
present = ("torch",)
|
||||
|
||||
|
||||
1486
static/index.html
1486
static/index.html
File diff suppressed because it is too large
Load Diff
1781
trainer_server.py
1781
trainer_server.py
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user