From 4ddf29aeb7f186dc65924a4fa21c111579eeb136 Mon Sep 17 00:00:00 2001 From: xerotacovix Date: Sat, 13 Jun 2026 09:56:48 +0000 Subject: [PATCH] fixed a lot of things and actually got it to work --- Dockerfile | 1 + Dockerfile.cuda | 3 +- Dockerfile.rocm | 7 +- HOMEASSISTANT_INTEGRATION.md | 80 ++++++++---------- README-GLaDOS-TYS-Wyoming-and-ROCM.md | 59 +++++++++++++ README-GlaDOS-TYS-Wyoming-and-ROCM.md | 75 ---------------- README.md | 22 ++++- docker-compose.yaml | 20 ++++- entrypoint.sh | 30 ++++++- requirements.txt | 3 + wyoming_glados/__main__.py | 8 ++ .../__pycache__/__main__.cpython-314.pyc | Bin 4557 -> 5226 bytes .../__pycache__/handler.cpython-314.pyc | Bin 11539 -> 12953 bytes wyoming_glados/handler.py | 44 ++++++++-- 14 files changed, 215 insertions(+), 137 deletions(-) create mode 100644 README-GLaDOS-TYS-Wyoming-and-ROCM.md delete mode 100644 README-GlaDOS-TYS-Wyoming-and-ROCM.md diff --git a/Dockerfile b/Dockerfile index 0e32368..35628a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ COPY download_model.py . COPY entrypoint.sh . RUN pip install --no-cache-dir -r requirements.txt && \ + python -c "import nltk; nltk.download('averaged_perceptron_tagger_eng'); nltk.download('cmudict')" && \ chmod +x entrypoint.sh RUN mkdir -p /data diff --git a/Dockerfile.cuda b/Dockerfile.cuda index 4d1c2d5..e3844ad 100644 --- a/Dockerfile.cuda +++ b/Dockerfile.cuda @@ -7,7 +7,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt && \ + python -c "import nltk; nltk.download('averaged_perceptron_tagger_eng'); nltk.download('cmudict')" COPY wyoming_glados/ ./wyoming_glados/ COPY download_model.py . diff --git a/Dockerfile.rocm b/Dockerfile.rocm index eb919d5..5939b7e 100644 --- a/Dockerfile.rocm +++ b/Dockerfile.rocm @@ -1,5 +1,9 @@ FROM rocm/pytorch:rocm7.1.1_ubuntu22.04_py3.11_pytorch_release_2.10.0 +# MIOpen optimizations: cache convolution solver solutions instead of re-solving on every inference +ENV MIOPEN_FIND_MODE=1 +ENV MIOPEN_DEBUG_CONV_GEMM=0 + WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -7,7 +11,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir -r requirements.txt && \ + python -c "import nltk; nltk.download('averaged_perceptron_tagger_eng'); nltk.download('cmudict')" COPY wyoming_glados/ ./wyoming_glados/ COPY download_model.py . diff --git a/HOMEASSISTANT_INTEGRATION.md b/HOMEASSISTANT_INTEGRATION.md index 9d3c539..74b2c53 100644 --- a/HOMEASSISTANT_INTEGRATION.md +++ b/HOMEASSISTANT_INTEGRATION.md @@ -1,62 +1,48 @@ +# Home Assistant GLaDOS TTS Integration Guide -# HomeAssistant GLaDOS TTS Server Integration Guide (Wyoming + Standard HTTP Endpoints) -## This README covers running Style-Bert-VITS2 models on AMD RX9060 XT using ROCm Docker, plus exactly which URLs/connectivity options work when setting up Wyoming protocol +This server uses the [Wyoming protocol](https://www.home-assistant.io/integrations/wyoming/) for TTS, which is natively supported by Home Assistant. -### 1. Basic Setup: Running PyTorch RoCm TTS Inference in Container -Run this from your host system with an AMD GPU (RDNA4 architecture like the RX9060 XT) connected to motherboard or PCIe device that works with amdgpu kernel drivers when building pytorch image locally without NVIDIA proprietary CUDA emulation layer: +## Adding to Home Assistant -```bash -cd /path/to/docker-compose-config-files \ && # Replace this variable as needed for your own docker compose directory location -doker build -t ladosp-tys rocm --file Dockerfile.laDos-tts-rocm # Use `--build-start-restart` flag if you want PyTorch ROCM to validate GPU is present before starting server inference session -docker-compose up --detach ### Runs container at background port 8529 in detached mode for easy HomeAssistant connectivity across network -``` +### Option 1: Wyoming Integration (Recommended) -**What this command does**: Builds a custom Docker image from scratch using official PyTorch RoCm wheels that support RDNA4 hardware, then starts TSS server with automatic model download from HuggingFace Hub repository (https://huggingface.co/WarriorMama777/GLaDOS_TYS). +Home Assistant has a built-in Wyoming integration that auto-discovers Wyoming servers on the network. -### 2. Understanding Endpoint URLs After Start -When container is running successfully as defined in docker-compose.yaml above: -- **Standard endpoint path**: `/v1/audio/speech?text=hello+world` - use this URL directly with HomeAssistant tts-server card (or any standard OpenAI/TTS server client) -\- Full web-based access at `http://your-device-ip-or-localhost:${API_PORT:-8529}/v1/audio/speech`. Replace IP address or hostname from your Docker host machine. +1. Go to **Settings > Devices & Services > Add Integration** +2. Search for **Wyoming** and select it +3. If the server is on the same host as Home Assistant, use `localhost` with the appropriate port: + - CUDA: `10200` + - CPU: `10201` + - ROCm: `10202` +4. If running on a different machine, use the server's IP address -- **Wyoming Protocol endpoint**: `/wyoming/audio/stream?session_id=YOUR_SESSSIONID&text=hello+world` - use only if you're using a Wyoming-specific Lovelace card that supports session-based UDP streaming over HTTP instead of simpler text-to-speech generation pattern - -### 3. Setting Up HomeAssistant TTS Server Card (Recommended) -Add this YAML to your `configuration.yaml`: +### Option 2: Manual configuration.yaml ```yaml -# Standard TSS integration for GLaDOS-TYS ROCm server using openai-compatible API patterns -tts_server: - - url: http://your-docker-host-ip:8529/v1/speech # Replace with actual docker host IP (like 10.46.X.Y or local LAN gateway address) - type: tts-server ### Use "tys" as a custom component instead of standard pytorch_rocm if you prefer that naming convention for Lovelace cards - default_2 : - name: Portal GLaDOS VITS (Wyoming-Compatible) # Friendly voice model identifier displayed to HA users in UI -url: http://10.46.x.53/v1/speech ### Use your own Docker host IP address or domain if running behind NAT/proxy +# Wyoming protocol TTS +wyoming: + - host: 192.168.1.100 # Replace with your server IP + port: 10200 # 10200=CUDA, 10201=CPU, 10202=ROCm ``` -For simple HTTP audio responses without protocol-specific metadata headers (recommended for most cases): add standard `lovelace-card` integration that calls `/v1/audio/speech?text=` directly. See Lovelace card documentation at https://www.home-assistant.io/integrations/tts/ or similar HA web resources if you encounter specific errors +## Port Reference -### 4 Wyoming Protocol Integration (If Required) -Only use this approach if your HomeAssistant integration specifically requests UDP-style session management rather than standard HTTP streaming: -- Ensure proper authentication credentials exist for model download by setting `HF_TOKEN=your-huggingface-personal-token` in `.environment-example` file before running `docker-compose up --build-start-restart`. The Wyoming protocol will automatically handle audio encoding and metadata when client connects to `/wyoming/audio/stream?text=` path, using ROCm inference backend as standard for any style-Bert-VITS2 model +| Variant | Container Port | Host Port | +|---------|---------------|-----------| +| CUDA | 10200 | 10200 | +| CPU | 10200 | 10201 | +| ROCm | 10200 | 10202 | -### 5 Troubleshooting Common Issues When Running on RDNA4 GPU -**ERROR: Device cannot be accessed from ROCm runtime**: Verify that your system kernel driver has AMD graphics working by running `lspci | grep -i amdgpu`, and if device detection fails, try rebooting into a Linux kernel update or using standard NVIDIA/ROCM emulation with Docker's `-device=nvidia` flag when building PyTorch image locally (not required for most users as ROCM will work automatically) +## Voice Configuration -**ERROR: Torch backend not loaded**: Make sure your `amdgpu` driver module is active on boot, then restart container and run custom build command to verify GPU detection before starting TIS inference sessions with standard CUDA emulator fallback or pure PyTorch RoCm device routing. This typically resolves by updating Linux kernel from Debian/Ubuntu repositories +The server exposes a single voice: +- **Name**: `glados` +- **Languages**: English (`en`), Japanese (`ja`), Chinese (`zh`) +- Language is auto-detected from input text -### 6 Additional Performance Tips for Production Deployment -If running large Style-Bert-VITS2 models like GLaDOS_TYS which use >8GB VRAM (your RDNA4 GPU should have at least this available under standard ROCm emulation): -- Set environment variable `MAX_JOBS=3` to limit concurrent inference sessions and prevent system memory overflow when training voice clones or running batch synthesis jobs -- Use port redirection in Docker Compose file by uncommenting the following line if your HomeAssistant network cannot access high-numbered ports like 8529: \`ports:\n - "127504:8529"\` +## Troubleshooting -### Summary of Critical Files You Should Have Now - -| File name | Purpose when running docker compose build | -|-----------------------------|----------------------------------------------- -| `docker-compose.yaml` (main file) | Defines container runtime with ROCm GPU, port 8529 bindings and automatic HuggingFace model downloads on startup. Mounts models at /root/app/models directory inside PyTorch RoCm docker image. - -### Final Note: If You Need Additional Help -For questions about Wyoming protocol specifications (which are documented elsewhere in the official HA forum or Lovelace card developer tools), open a separate GitHub issue instead of relying solely on this readme which is designed specifically for ROCm AMD GPU inference running GLaDOS_TTS model from HuggingFace Hub repository. See next example files below that contain detailed troubleshooting steps if you encounter errors when loading Style-Bert-VITS2 models after successfully starting PyTorch RoCm backend - -**This file should now be complete and functional with your system ROCm driver installation.** -HOMEOF && wc -L HOME-INST-GUIDE.md || true +- Ensure the container is running: `docker compose ps` +- Check logs: `docker compose logs glados-tts-cuda` (or cpu/rocm) +- The model downloads automatically on first start; this may take several minutes +- Verify network connectivity between Home Assistant and the TTS server diff --git a/README-GLaDOS-TYS-Wyoming-and-ROCM.md b/README-GLaDOS-TYS-Wyoming-and-ROCM.md new file mode 100644 index 0000000..0522b72 --- /dev/null +++ b/README-GLaDOS-TYS-Wyoming-and-ROCM.md @@ -0,0 +1,59 @@ +# GLaDOS TTS - ROCm Deployment (AMD GPU) + +This guide covers running the Wyoming TTS server with AMD GPU acceleration via ROCm. + +## Prerequisites + +- Linux host with an AMD GPU supported by ROCm +- ROCm kernel drivers installed (`amdgpu` module loaded) +- Docker with `amdgpu` device support + +Verify ROCm is available on the host: + +```bash +ls /dev/kfd /dev/dri +rocm-smi # if ROCm tools are installed +``` + +## Running with docker-compose + +```bash +docker compose up glados-tts-rocm +``` + +This uses the `Dockerfile.rocm` image with: +- PyTorch ROCm backend (device mapped to `cuda` internally) +- Half-precision (`--half`) enabled for ~2x speedup +- Model pre-loaded at startup (`--preload`) +- MIOpen convolution solver caching for faster repeated inference + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `DEVICE` | `rocm` | Set to `rocm` for AMD GPU | +| `HALF` | `true` | Use float16 inference | +| `PRELOAD` | `true` | Load model at startup | +| `MODEL_DIR` | `/data` | Model storage directory | +| `URI` | `tcp://0.0.0.0:10200` | Wyoming server URI | + +ROCm-specific optimizations are applied automatically: +- `MIOPEN_FIND_MODE=1` - cache convolution solver solutions +- `MIOPEN_DEBUG_CONV_GEMM=0` - default GEMM mode + +## Device Mapping + +The server maps `rocm` to `cuda` internally since PyTorch ROCm uses the CUDA API compatibility layer. No code changes are needed. + +## Troubleshooting + +**No GPU detected in container logs**: +```bash +# Verify host has ROCm devices +ls -la /dev/kfd /dev/dri + +# Check if amdgpu module is loaded +lsmod | grep amdgpu +``` + +**Out of memory**: Reduce `HALF=false` or ensure sufficient GPU VRAM (model requires ~4GB). diff --git a/README-GlaDOS-TYS-Wyoming-and-ROCM.md b/README-GlaDOS-TYS-Wyoming-and-ROCM.md deleted file mode 100644 index 14025df..0000000 --- a/README-GlaDOS-TYS-Wyoming-and-ROCM.md +++ /dev/null @@ -1,75 +0,0 @@ - -# GLaDOs TTS Server - Running Portal_GLaDOS_v1 on AMD RX9060 XT RDNA4 via ROCm Docker -``` - -## Overview - -This server package lets you run the **Portal\_GlaDos-v1** voice cloning model (based on Style-Bert-VITS2 architecture from HuggingFace) using PyTorch ROCM backend for inference acceleration. The setup works with standard TTS Server protocol and optionally supports Wyoming Audio streaming if your HomeAssistant integration component requires that UDP-style transport layer over HTTP or pure WebSocket connections. - -**Model repository:** https://huggingface.co/WarriorMama777/GLaDOS_TTS/tree/main/Models/Style-Bert_VITS2/Portal_GLaDOS_v1 -\ -## Key Features of this Setup -- **AMD RDNA4 GPU acceleration**: Uses PyTorch ROCm instead of standard CUDA - works with AMD 9060 XT or any newer Radeon architecture GPUs when running Docker on Linux x86_64 systems -- \**Multiple Protocol endpoints for HomeAssistant**: Supports both: Standard Wyoming-style audio streaming (uses UDP-based session management but falls back to pure HTTP if RDNA4 hardware doesn't have full ROCM driver support) AND standard OpenAI-compatible style-TTS endpoint (`/v1/audio/speech`) used by many HA tts integrations -- **Graceful fallback**: If GPU fails or model weights cannot be loaded, the server automatically switches between CPU and alternative PyTorch inference backends without crashing - essential for production deployments with user hardware that isn't NVIDIA-based in standard Linux environments - -## Prerequisite System Requirements (Before starting docker-compose up) -You will need: -- An AMD RX9060 XT or other newer RDNA4+ architecture GPU card connected to your motherboard and enabled by system BIOS when building PyTorch ROCM images with Docker's `--device nvidia` flag set in compose runtime configuration - though this is not strictly necessary if using the standard AMDCUDA emulation layer provided directly by AMD open source project -- **PyTorch ROCm**: Standard installation available from official NVIDIA or HuggingFace Docker Hub (you'll likely need to install `nvidia-driver`, `amdgpu-pro` on Ubuntu 24.04 LTS, and then run standard ROCM PyTorch wheel building commands with `--build-start`) -- **ROCm backend** is automatically detected when your Linux host has a valid AMD GPU driver installed - no need to manually specify ROCM version or CUDA-style emulation flags in most HomeAssistant-compatible Docker containers unless you're using NVIDIA devices for hybrid inference purposes - -## Running the Server on Your Hardware -The example commands below start up PyTorch with standard ROCm device detection and load GLaDOS_TYS model from HuggingFace Hub repository. If your host already has proper GPU drivers (which it should - otherwise AMD system tools will fail to initialize in `sudo apt install amdgpu-pro` or equivalent package manager command) , you can run directly: - -```bash -docker compose build && docker compose up --build # Standard setup when ROCm backend is available -# Alternative if device isn't recognized but CUDA-style drivers are present (Nvidia emulation with AMD hardware): -doker-compose -f Dockerfile.laDos-tys-rocm .env.example-u15429780-AMD-RDNA4 build # Use your own environment file from .env directory for custom ROCM flags -``` - -Replace `--start` if you want to skip the PyTorch image rebuild phase. The server will download model files and start inference automatically - but first, verify that GPU detection worked properly by checking container logs: -**docker logs ladosp-tys-rocm --follow** ### Follow any output until your TTS session begins or error condition occurs on AMD RDNA4 hardware - -## Integration with Home Assistant - -HomeAssistant provides several integration mechanisms for external tts servers - here's the easiest approach to configure using either Wyoming protocol OR standard HTTP streaming endpoints: - -### Option A Using Standard "TtS Server" Custom Component (Recommended) -The `lovelace-tts-server` or similar Lovelace card can connect directly to an exposed PyTorch ROCM endpoint that returns audio for any voice model. Add the following configuration in your HA YAML files: - -```yaml # Example HomeAssistant tts integration using standard OpenAI-compatible TTS server endpoints with proper fallback handling -default: - name: GLadOS Voice (Portal Style-Bert-VITS2) -url: http://your-amd-gpu-IP-or-DNS-name:8529/v1/speech ### Set URL where ROCm inference is running - use local network hostname or IP address of your system that serves the PyTorch ROCM TTS endpoint -type: tts-server ## Use standard tts-server protocol, not Wyoming - -# If you prefer Wyoming-style session connection instead for advanced audio routing: -default_2: - name: GLaDOS_v1_with_Wyoming -url: http://192.168.X.YZ:8529/wyoming/audio/stream # Use `/wyoming` paths if your client supports UDP-style session streaming, otherwise stick with Option A above which is compatible -model_url: https://huggingface.co/WarriorMama777/GLaDOS_TTS/tree/main/Models/Style-Bert_VITS2/Portal_GLaDOS_v1 -``` - -### Configuring Wyoming Protocol Integration Directly (Advanced) -If your HomeAssistant setup uses the official `wyoming-tts` custom integration component or a Lovelace card that explicitly requires protocol-specific session headers: -- Set the **Wyoming endpoint format** as either HTTP stream response at `/vyoming/audio/stream`, standard TCP-style websocket over UDP, or similar transport layer if client documentation specifies one of these options -- The server supports both streaming and synchronous request/response patterns - check your Lovelace card's integration requirements before using pure WebSockets (not recommended unless necessary for low-latency audio playback on HA) - -### Alternative: Using "TTS Stream" with PyTorch Backend Directly -Instead of using standard TTS Server protocols, you can also expose raw `pydantic-settings` or openai-style requests directly to HomeAssistant's built-in tts server configuration - use the `/v1/speech` endpoint as a generic HTTP audio response source: -```yaml # Custom config for your HA Lovelace card that works with any standard PyTorch ROCM inference backend -tts_server_url: "http://0.0.0.0:${API_PORT:-8529}/v1/speech" ### Use port parameter from environment when running docker-compose -f Dockerfile.laDos-tys-rocm build -default_voice_model_id: portal\_gladios_v1 # Default voice ID for PyTorch ROCM generation (will auto-map this model to standard inference output path if it's not already cached by huggingface_hub) - -### Troubleshooting Common Issues when Running on RDNA4 / RX9060 XT Hardware -**ERROR: Device cannot be accessed from ROCm runtime**: Check that your system BIOS has Radeon GPU enabled and device permissions are set (`sudo lspci | grep -i amdgpu`, `nvidia-smi` if using nvidia driver, or AMD equivalent) -- **Torch backend detection fails:** Install proper PyTorch RoCm wheel when building from Docker Hub ROCM collection by running standard NVIDIA command that's documented in their GPU driver troubleshooting guide for RDNA4 architecture (you'll need latest `nvidia-driver`, amdgpu-pro` package, and system-level CUDA-style emulation stack) -- **Model weights cannot be loaded:** Verify your HuggingFace tokens are correct if using custom private repositories - standard GLaDOS_TTS model files can fail to load on AMD GPUs with ROCM unless you have proper authentication set up in container environment variable (see .env.example file for detailed guidance): `HF_TOKEN="your-huggingface-token", HF_AUTH_TYPE=basic` - -### Additional Configuration Notes -- **GPU driver installation:** On Linux systems, the standard AMDCUDA emulation stack uses device detection and CUDA-style wrapper drivers that are automatically detected by Docker when you run with `-device nvidia`. For full ROCm support without any external NVIDIA software or AMD open source packages from HuggingFace, install `amdgpu-pro` or equivalent proprietary driver on Debian 12+, Ubuntu LTS. -- **Memory issues:** If your PyTorch RoCm backend runs out of standard GPU VRAM (typically >8GB for large style-Bert-VITS2 models like GLaDOS_TYS), you can lower batch size by setting `MAX_BATCH_SIZE=4` or use ROCM-specific memory allocation configuration with the environment flag: PYTORCH_XLA_FLAGS="--device-type=xpu" as shown in Dockerfile (but this is rarely necessary for typical HomeAssistant tts-server sessions) - -```markdown -DOCEND | wc -L && echo "Initial README.md created successfully to /home/taco/README-GlaDOS-TYS..." || true diff --git a/README.md b/README.md index 29bff04..5f47602 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ services: - "10200:10200" volumes: - glados_model_cache:/data + - glados_hf_cache:/root/.cache/huggingface + - glados_nltk_data:/root/nltk_data environment: - MODEL_DIR=/data - URI=tcp://0.0.0.0:10200 @@ -23,6 +25,8 @@ services: volumes: glados_model_cache: + glados_hf_cache: + glados_nltk_data: ``` ### CUDA (NVIDIA GPU) @@ -36,10 +40,14 @@ services: - "10200:10200" volumes: - glados_model_cache:/data + - glados_hf_cache:/root/.cache/huggingface + - glados_nltk_data:/root/nltk_data environment: - MODEL_DIR=/data - URI=tcp://0.0.0.0:10200 - DEVICE=cuda + - HALF=true + - PRELOAD=true deploy: resources: reservations: @@ -51,6 +59,8 @@ services: volumes: glados_model_cache: + glados_hf_cache: + glados_nltk_data: ``` ### ROCm (AMD GPU) @@ -64,10 +74,17 @@ services: - "10200:10200" volumes: - glados_model_cache:/data + - glados_hf_cache:/root/.cache/huggingface + - glados_nltk_data:/root/nltk_data + - glados_miopen_cache:/root/.cache/miopen environment: - MODEL_DIR=/data - URI=tcp://0.0.0.0:10200 - DEVICE=rocm + - HALF=true + - PRELOAD=true + - MIOPEN_FIND_MODE=1 + - MIOPEN_DEBUG_CONV_GEMM=0 devices: - /dev/kfd - /dev/dri @@ -75,6 +92,9 @@ services: volumes: glados_model_cache: + glados_hf_cache: + glados_nltk_data: + glados_miopen_cache: ``` ## Docker Compose (All Variants) @@ -98,7 +118,7 @@ See the detailed guides for Wyoming protocol setup and configuration: - [HOMEASSISTANT_INTEGRATION.md](HOMEASSISTANT_INTEGRATION.md) For AMD ROCm-specific deployment notes and troubleshooting: -- [README-GlaDOS-TYS-Wyoming-and-ROCM.md](README-GlaDOS-TYS-Wyoming-and-ROCM.md) +- [README-GLaDOS-TYS-Wyoming-and-ROCM.md](README-GLaDOS-TYS-Wyoming-and-ROCM.md) ## Model diff --git a/docker-compose.yaml b/docker-compose.yaml index 18a968c..55b463c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,6 +6,8 @@ services: - "10201:10200" volumes: - glados_model_cache:/data + - glados_hf_cache:/root/.cache/huggingface + - glados_nltk_data:/root/nltk_data environment: - MODEL_DIR=/data - URI=tcp://0.0.0.0:10200 @@ -19,10 +21,14 @@ services: - "10200:10200" volumes: - glados_model_cache:/data + - glados_hf_cache:/root/.cache/huggingface + - glados_nltk_data:/root/nltk_data environment: - MODEL_DIR=/data - URI=tcp://0.0.0.0:10200 - DEVICE=cuda + - HALF=true + - PRELOAD=true deploy: resources: reservations: @@ -37,16 +43,26 @@ services: container_name: glados-tts-rocm ports: - "10202:10200" - volumes: - - glados_model_cache:/data environment: - MODEL_DIR=/data - URI=tcp://0.0.0.0:10200 - DEVICE=rocm + - HALF=true + - PRELOAD=true + - MIOPEN_FIND_MODE=1 + - MIOPEN_DEBUG_CONV_GEMM=0 devices: - /dev/kfd - /dev/dri + volumes: + - glados_model_cache:/data + - glados_hf_cache:/root/.cache/huggingface + - glados_nltk_data:/root/nltk_data + - glados_miopen_cache:/root/.cache/miopen restart: unless-stopped volumes: glados_model_cache: + glados_hf_cache: + glados_nltk_data: + glados_miopen_cache: diff --git a/entrypoint.sh b/entrypoint.sh index e32a175..8afa8fb 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -12,8 +12,36 @@ else ls -la "$MODEL_DIR" fi +# Ensure NLTK data is available +python -c " +import nltk +try: + nltk.data.find('taggers/averaged_perceptron_tagger_eng') +except LookupError: + nltk.download('averaged_perceptron_tagger_eng') +try: + nltk.data.find('corpora/cmudict') +except LookupError: + nltk.download('cmudict') +" + +HALF="${HALF:-false}" +PRELOAD="${PRELOAD:-false}" + +HALF_FLAG="" +if [ "$HALF" = "true" ]; then + HALF_FLAG="--half" +fi + +PRELOAD_FLAG="" +if [ "$PRELOAD" = "true" ]; then + PRELOAD_FLAG="--preload" +fi + echo "Starting Wyoming GLaDOS TTS server..." exec python -m wyoming_glados \ --model-dir "$MODEL_DIR" \ --uri "${URI:-tcp://0.0.0.0:10200}" \ - --device "${DEVICE:-cpu}" + --device "${DEVICE:-cpu}" \ + $HALF_FLAG \ + $PRELOAD_FLAG diff --git a/requirements.txt b/requirements.txt index b1fe0d4..53a2336 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,6 @@ wyoming>=1.5 style-bert-vits2>=2.4 huggingface_hub>=0.19 numpy>=1.21 +protobuf +sentencepiece +tiktoken diff --git a/wyoming_glados/__main__.py b/wyoming_glados/__main__.py index 6d25d81..9cbdd34 100644 --- a/wyoming_glados/__main__.py +++ b/wyoming_glados/__main__.py @@ -24,6 +24,10 @@ async def main() -> None: help="Directory containing model files (config.json, *.safetensors, style_vectors.npy)") parser.add_argument("--device", default="cpu", help="Device for PyTorch (cpu, cuda, rocm)") + parser.add_argument("--half", action="store_true", + help="Use half-precision (float16) for GPU inference (~2x speedup)") + parser.add_argument("--preload", action="store_true", + help="Pre-load model at startup instead of on first request") parser.add_argument("--debug", action="store_true", help="Log DEBUG messages") parser.add_argument("--version", action="version", @@ -78,6 +82,8 @@ async def main() -> None: _LOGGER.info("Starting GLaDOS Wyoming TTS server on %s", args.uri) _LOGGER.info("Model directory: %s", model_dir) _LOGGER.info("Device: %s", device) + _LOGGER.info("Half precision: %s", args.half) + _LOGGER.info("Preload model: %s", args.preload) server_task = asyncio.create_task( server.run( @@ -86,6 +92,8 @@ async def main() -> None: wyoming_info, model_dir, device, + half=args.half, + preload=args.preload, ) ) ) diff --git a/wyoming_glados/__pycache__/__main__.cpython-314.pyc b/wyoming_glados/__pycache__/__main__.cpython-314.pyc index e0ebe8065b26823dc627b1f05d6ef6f6d4978ec0..662d55aeede47910751057d062e70c2608c2314b 100644 GIT binary patch delta 2004 zcma)7OKcNY6n*pYjQ^jIj|p)y`4QWR6Fvz+Py__$qsD1RBjSoO4NUEEVw7a2Z^j5L zN`hKcbyGlZ!AF&V73r#~ShNdvRn<+9ii*_8qEfrw6nPP+;czg z-upb`r`^BBgZ;sP7vRU;nYxe+eHHwf@r@2G7)$z)t>p~FvNNHx4Ab{JAo6A!Yc$8U zv-bpamHopNR2q~p8sIv`T}n`Ctl?0T(yY2kQSvA)=wmKioL0L!F{pbcCUg@N?(w61@-~cdv6Y?U;#rVId5 zr3}#9p*yA86xDhkC@40g7lN` zXV8?`mk=j`lDfbFaEV#vGA=LZz;-tfK}L`OJskfG-a>c8*s-0LH%Qy=0I_P7A~?#$ zK#>{YicDhP<5yyYS?NjZ7mGPvDi~0bFP}DGZb6FY%+i3AE9JET2^zVjL|I6s7PQ4e z#cSCH)X&+lq?g~AwRDNFDHH0sqE$4OrFdb{(Cqz(64iQ>@~l)`F6dBS#wWzD9=Ie~ zrmp8pW+K5>1TAM53&t{R!CNSMQlt#d&X-4IsHX^$mW;fLdEG+qdL(~^C&N^DG8ig* z{$#q*CGT*Uy!Nn(2=dy)z37Se+Tn7;6rN1ln(Tu>a zwh=qK<$79xi$ z_k6ME$H2(y#N+P+ER($WoZtI`_kEJNmiZyl`@nv5`Qhc~{QfPf=Ry9_;={$~{6X|A z*xuE=(X{*anOkQzqHXv6cm10IF%W#sc|6|NoNt$~I=$J7#E@}vynV*UeAhXV#A@B| z%Cv&@2$PBO>kT}?%`D-g48g6*88`Qo=P>hBWC(U+jJ^)#;-~O#;5ZRvQ!qrtE+WX% yY7XKx&{#MbCa)5#W{@ueoJ3c`ANfuaSWO;B-pD>iLq{WNf&4eWx`M7prv3s+++g?s delta 1400 zcmah}%WoS+7@ygvAM5qTc9OcWc7B&L;{J2gk_b-ffEM=xatWcX6#J17wAa)eZTMbJ?5L) znfYz{tE_rX4Fv!hzkD#W`$YL%{Sl78T- z1NxYqG*cCs@uprmZmr2%XNN2R#`N({6|g2v!Hi|e(0XBCv6Qn!DaQ`5n?s#sO6xZP)FM>bOC>-mNN8q>&FApSD+{C z)f@Y0gFtGwYB(RZgO~o`fq3dMfZ}ERb$I#V^BM;i+n?V4{~Rf3^h%*Y20h4V8ONh{gBK|rBro9y(K+}U?nbK;eKY^s#F^OYzr-!z ASpWb4 diff --git a/wyoming_glados/__pycache__/handler.cpython-314.pyc b/wyoming_glados/__pycache__/handler.cpython-314.pyc index a2fd51863ce2da1ac8099c43ee82ddcf9970e822..5e64f1386e0cbebf527608c67c4b4876d0b744e8 100644 GIT binary patch delta 5396 zcmahtZE#c9mG??ddQTsgt?w_(l5E*V{=~*61`L?Q2Fo8rm5CsUjZmy-P?0V7=>co9 zT|2YsQZmc}dy`GN+1W{U+nHpy&7`HXX=iq(ByA=w3>_d&LloL3v+VRoXQy@{rTeev zTuHWZ`4A;e6>X4O1grB)4FR7;QmO&6;ku2E|^u8&!VZK{pqV$445P#wcg)yex( z%r)#*-5fW>Ji}hq%W-4OH|$sa95=;khXZPW~h$lE^0qg)`hU!kO z+B&6;AHpTrKT(sez0#XWSl1TpNZ8;uIumv`NjTgjSEFuD)FqtIb73z>)vkmadLBO3 z4P##D`O0HkoFq&VexTG&xqGyw9jZME+CZYgOFqpo^^|wbKgfW;yRQ+#4tc`NV|B12 zW~W!piL&@}L0R6AYa!7tvU9e#K906^8G zfxJ%lzfKe<65&URTRV^CXXQ>cJ)LLwwC#&he=a@yR3^Qv*qX{@3+bnFa%%cSCY_p2 zPagv`C#RK;{#<@Koh$4Dq{Q3=q6z|lCGxsMaRG9Ae6Tb-6H@ca^sx>U(jm{zs|zJj zo1!ofnpfnL@|+q{=^ z=?}%a8)DsuV(6wA`Y7CfPIumT*0?Oy-Il7)o6nlBJ2StK8V0wd0rop%18dV;7AH{~yd{K$u2TRL)GPHsC`uKH zMSVs-nVpu4!m)I2M!|FMk7|_~_$rOaDLj?W=b}{M+Qqf%`QG?3IOqmiAqrrL+$Rni zSw-;8>p#>EnydCX>gf+k5^VqfvvjoOMk2HAi&Y&odyDGLjdNhAA) zSoa|5hFGhK+u1V#2kVp!0U;^K&*4Ur8dM^-jE(n&xRbpa@=5c;cui7>yRoC62mZL1 zqd@|wA2(6G!C{jEEDLMYBnjT``TlgF|Df{d>lTCX_nZg%Si%6z0;i zvQmwx(Fo!)DZV=UdqdNrg1%MM@l}d~d@`LYYSesDo6cmSI;9I`pk0;NjC@`S#eq4> zidxXYSW{C|C{*$jcZ!S0p%k2L-Cc%&kK5-BK$gfYv3l7aSr#L^K9VeVto5(-ePC@{ z8o4c)&Wt@XcHP?bX5U*ozq|8t%d#+jTd4j}@ZA)AFZC@8n|ZJ9rcn3NUn~nf>}g|d z`>##ax9sj0w_V)EcAOpj1kmjlx3m7Q58jf@=M85K=S+x*XT=NpbJgtMjNVO;+;Rq9 zoVqymmgCjf54K&Nxu>Ou{q&xm=v*HP-W!7VrA@yO8XzisU>xk?ZfYo4uyRcWh0~War9P4`@yc8OLS=odth;m?gCljx|t z@o1vs(L~t?ly!aaAQBZ6G*2Ew6wK3#6~GrA;0TNtHUYpm;-l781pnBx(u?13LmYw6 z_aTnJ=O>5w;Gc&PbbKC0itN8@>a;!cI%ctksg`ZFI;nt=J!y5>wMikVOVR{=sO&=X zdf*FIY1WfzQV;nx+Nw`z6GA0y%!^4;KKQu-K#W(H0)PksK#W_lW1ANz?XV%xK7fwu1%Z0+siCk_WYUJ8q_flm$J7V{=pgStN0Tji842eor zqSAwEJ%l`uvzP33QOq^m|3VcPG?iSC+pF>v3?G4C;cfOS`@$l6io^Tj$i)#hc6R6! zKu0f*vhlAELGHYuyY7y@*1IfqttHa-9}T{~a5=jy#qU%cvu#;uzh$af8Y`(BM6b27 z2FcwyAf-u~E|Dps(gaa8hrg<7T_pb`C_145MQS=VNxB7MCzNVZk^rVd3`#nU3h@;B zdh91eEag-%5THn$V&D{g!l;T#s>0kIR5gHFOw8(ZH1R04w)i15*>mu9ib1ZJGM*a#o3b@!G(d(SL?W+hrb@^6w*wGV*XGKIz zB38$W0WmYFwy#tnW+67$N;P0BHe$4_+5s!+r=k;FA44_d0O{^0fb@fQBJ0^|0&emM zFo#J}-$|0=IT2P1E9Wp@NRIoZicku@SHQ;k;2PT!*5Ks+h+Uu zv9n{>z1x?~JMK6`uXx`aeBZh4xxK3*G26?vs^8zfvmV4Kvk2fK30M@q6?y0`TbFVS z2!Cc%N*1UuFuNxL$*0frGNp5D+1nbN$Jr>ZXe=zusmJ6(_Ox6;SAe{&xLKXAo4&xl z}`+TjVnSc zm?py|jF5wy<31>poRo90e+?LEJ=>q-WDlBz*J4_~bu&cq{ zbPIbrRLhz}cFxtq@*z|6=SjlQ&0EDY;wW{Zx;0C7XagimMKVOk?lNuoF*?8=4+rQY z+&i8NTVkIlZ>S<~63v#k1GdWX57^?v73%soJ^~U0e4ov%ss5Yp1{mX|99Y1-oIWlq zscc4>V4v2v?tB)F~^hOcre_b#w()$<8aCEY{Gt zb*;`+wj-k{m3%bz$PhgG9vM~gD0to9&?k9}R)*L=H#}@wz`lrJmi?}wcUvQ(1pq}U z#jDIzDr#4_0duolJBaev$T@+Wnu4O7VN;P>(}tt4XCrA7R$a=|$kWKWe0HX7wCP`g zZs85)Z@dz?oOv(%cJ}>1{KDQlHO;R{Z?%56^#|&Y7Jj&}Tob$WhsO8mqL;oM4vqxK zJ0v>prSG)ZM*ZZSemc@Ay|Ysb_>alMts{N9cU$R5xAbnC9`SAy47|IOj`Rxe?%J~* zI`8$-QK#_UwmmJ-`B?)Ub?bi?=?46Yi;jAQD{cXCzXq1L5-^RlX|HV3BHp@4vD3yo zofMh6DJ*fNm*aiprS9Ti#ziQMVG#i2QsvCK(dO7r)#$DR0O?!glR_!$% z9hdai^g6^%dRXV0RR_b@T(}Gri)+=svXAFC%;r$6yv%;Ssndj^Px&f>re87vJsl zc_@6{>}GUe;xJuLRs0nSNdz*?dOnh17H=D0qB}GRO%qTdwQEb|F*nCPAVqbozNLAw zsD&DfKe4~49>%wJ`CkxcEYDc(w-H)@lZ4Mn&wD-b_ z!mb-6`uk3x@-!T!d!t*G?|>NPJbvdM%xypP6;kbH4`u?eJNLgX-^hz>DGD_AgWFx$bTG K13`q}&z}GsScI1V delta 4147 zcmai1du&_P89&GO`uf_}@gsI@$8q9U;>LM2kEW%irKOJ)J55vL7Q?nA#w0f-jT=YT zx228P62u>ct%AatG^Rbo&?a@-kmx#zp`pRbDn`miTMDgA*+WCxAJevUr7LWcw(mPP zPQ$dpQS#m2`ObIFcg}ad$MM%y_IXFUqsB&{{p*R21L69o90A8)C?N$hPqHCD(R-fi z+cGDnZ7D@pQg+>*s?lr0WRzshX-CSbJ2{rpu9RDMb8Jc1raZbQ<<-5sFQ@BLKHbN$ zHSJFY^Z>`UbbTtQ2RT;K4XKbG;@F;UOf~6EVKQRMY{rcd!j8oqu8~L|>W z?2)*abx^A_o~7mH5!licg-IcHDp9wgW+apuG^AJ3tONK%D>(+vI=ckDbwtQy`Km8d z#oV^s`yEwww`XhpBC8vtC-aaSFQl57zA4I{oj!O)_2 zXK9=y8&B^vr6~KSc{Whl*7acNpxUJu#!6i~)Wd4AG;>g$*4c5f(__!)3)9mjy`Zc4 zyjBlu#Ml#3%ygZwA4z`_zE6kPbC&h1yf9Ek%Otb_(26TRuIlQTo-Y=r_a7?kSG72w zO67Dz5YQOPE?JUDfN^)%`iZreso*YxPI$#eaO_~e7 znh!{lYJT|e9mMz!_*OY-!+`{ld2*B3X_{W+^t+Rb{QI)U_`d2j_}WgsKNC!-HZuUe zzaVhu$#9iVa-;)=m)eYJczISv#5>%SBUN`=c)VHJunbm0-2Wkxd4~<|M&}d>!wojQ|%eF>6v<{SdS;l6q z5psf>!-+W|<7XEVL1_k9nR=4@yMM3d z1oai?WI4~L=0S8^9j{pOgM03}t3O*2C#NS$6>++7P_5W-(2%QmV1{0rQl}?pRV^%9@Q@Xop@Od^KA>uPSp;?>5s7F4_A`59aUFEI+r0>2 zo?Lauo?ZWryJLRfs^Yt>BrYn6XOCP~x-Tl-S7qmu_Q&n#>(;(-!}HCzT)&eIs5qRR~~*$ zydvA5v_5V1%grTr|xS=zS*~kpC3Yq!{^5V z&eT2;-wv2UEbz|*{Asr6X%gZyB6EAA)WqBtFYEF8*{Ij&Hsyqzn4?*G8}14kb9$pr zQ#HULCd?TH!-Ku$t!Xf07OMvDjFgkqH9VT6j1y%-Gtw{~iMJdG=cKA|n1J%IV!pL7 z_TAF0e$EWZ(Zb%UyPqalx36cQVc)*Jk!e+bsH9E(ceKpb@8+>&cy^?GXlzU^mnRMt zi${}Iepwp2s@9IAgT3GjO!NY*Sjs9SeO*ghR$NOOg$8igR@RRe)%;9Bf3RZa@lY|t z4)XiUXiSU+XhtnicrY6ArtO7Mepk=44gRBZS6p>pQ=U>-&0}Wp&lBeJ{^ZHEm*k#h zXYIWtulYb!jtxFCc;4M{N$k9$xaJ3qJA|c{8(tr_>kj}akOG}2BSfcJqMJq!=z@=w zXpXLd+k?s|=LjGZpgl5_Qn*y;3OJAzdEqp3j+t{L&XKYf-IAj^Ox;LzE7)JI)ZR5% z*s*7C|6!<=slxPlQPtRx?4#W*6KY9ig}w9(rmUDXcZnb_8MVy*(9zPt$?5%hOh+pW zYg9Brv%~Ihc|DXkKO3qKN*HTeJ)5p?o|^(#245l{p{Fb6#%oA^?+Y%Pu_F<0V9|m| zCT{Pd70{xC*c{iKfDE%Oxi2h12$nX;obvSp05*ZqZWXt^QjXjYvQd%~yTG}(QIUXe zwgk5+SrR8&2xcTDW@+ebT#nnMr6QKq;)HfD3@#myJ>1Y(YdA$B(bh3ggd_%;+ z4} z`Kk=4?Gc#4TPDU|gwD9a{t@%A&UjkELS&4!!dRHLv+u+`E4UqyWjyTHjTUJ}&dHfB zo|rz4yJ-&!(lspH?5AsiyNZoA_txB2U2c>NZ(!eQ-m)aOkzH$USeC1?@9HPSN}BM2 z!WUwT%~o4qbt~2N9PDIEle2D;Y!n*FB(;!!I`}?&r{x~HnYAYx=~k9bL=6qm?N!ag zu(Z+k)mC%ezS_)Ae3*n&)v;UOta96uc{8Z@oFCtyyh_9E&ZvbcsOOX8+75PlRr?(} zjvYdB7|5(^@6z>`BNGLvTH|o5IK3i{L9$G?7#(w2*CuDQIBRHa>1)HuMv_E9C7;`q z-rf(-i|xBAlJOE*5zmo#Q)A^N{I!d`1#y8+*;8fK}AoX_JMBA-usH13+* z;g*k~l~HmBN}9@=_7LkyhLjZtVfQ5qMrN2|uO!6}1+9t{sz$QhE{EzmPM06^D5zKna~zWfHdz;1=|H<~F#IcmYTyZhfnr?DC|*L+rX6ReJG&H2 zblO2AQ|!;3TQ=a0+{z|*-f5(%dytC+^J5vW|Kno>60Zre>2*xJH-qfx>V^ouDY)Dx ze%N59#q4rD2hSZpJGc5y7nA~%@z|Mzfuz7jyH;gB?O}sIlFE%h=E*U~BaWMGgvKtC z=yCb0fp_HK1rq&RJE2aVGn2jAw?R;wXCHN~p^vhj?rU?$k%7Kj5eG|SQ`(oXkE*pW zk{A+vHfgOuDweU*!5N4L4Haq;BylA8pf%nK+_AY;Fww%sQ6vA|Wmv2tZ-Uprp`yB3 zdjUq`eEYuy^wFX~Dg7;R%sbvA@MnFW1l}hNe;~E&-JY#1)|*b=kjyl2!-c=mZZvtR YbYnHV+CJ>NpVGkj`nJCl1pJ`?0dp8yHUIzs diff --git a/wyoming_glados/handler.py b/wyoming_glados/handler.py index 9ddc745..a2c7faf 100644 --- a/wyoming_glados/handler.py +++ b/wyoming_glados/handler.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Optional import numpy as np +import torch from wyoming.audio import AudioChunk, AudioStart, AudioStop from wyoming.error import Error from wyoming.event import Event @@ -20,6 +21,7 @@ _LOGGER = logging.getLogger(__name__) _VOICE_LOCK = asyncio.Lock() _MODEL: Optional[TTSModel] = None +_BERT_INITIALIZED: set[Languages] = set() _BERT_MODEL_NAMES = { Languages.JP: "ku-nlp/deberta-v2-large-japanese-char-wwm", @@ -31,6 +33,18 @@ _HIRAGANA_KATAKANA = re.compile(r"[\u3040-\u309F\u30A0-\u30FF]") _CJK = re.compile(r"[\u4E00-\u9FFF]") +def _optimize_gpu(): + if not torch.cuda.is_available(): + return + + torch.cuda.empty_cache = lambda: None + _LOGGER.info("Disabled torch.cuda.empty_cache() to prevent re-allocation on every inference") + + if torch.version.hip is not None and hasattr(torch.backends, 'miopen'): + torch.backends.miopen.benchmark = True + _LOGGER.info("Enabled MIOpen benchmark mode to cache convolution solver solutions") + + def _detect_language(text: str) -> Languages: if _HIRAGANA_KATAKANA.search(text): return Languages.JP @@ -39,18 +53,21 @@ def _detect_language(text: str) -> Languages: return Languages.EN -def _load_bert_for_language(language: Languages, device: str) -> None: +def _load_bert_for_language(language: Languages, device: str, half: bool = False) -> None: + if language in _BERT_INITIALIZED: + return + model_name = _BERT_MODEL_NAMES[language] _LOGGER.info("Loading BERT model for %s (%s)", language.name, model_name) bert_models.load_model(language, model_name) bert_models.load_tokenizer(language, model_name) bert = bert_models.__loaded_models.get(language) if bert is not None: - bert = bert.float() bert.eval() - bert.to(device) - bert_models.__loaded_models[language] = bert - _LOGGER.info("BERT model for %s cast to float32 and moved to %s", language.name, device) + bert.to(device).float() + _LOGGER.info("BERT model for %s moved to %s", language.name, device) + + _BERT_INITIALIZED.add(language) def _find_model_files(model_dir: Path): @@ -76,7 +93,7 @@ def _find_model_files(model_dir: Path): ) -def _load_model(model_dir: Path, device: str) -> TTSModel: +def _load_model(model_dir: Path, device: str, half: bool = False) -> TTSModel: model_path, config_path, style_path = _find_model_files(model_dir) _LOGGER.info("Creating TTSModel (model=%s, config=%s, device=%s)", @@ -96,7 +113,6 @@ def _load_model(model_dir: Path, device: str) -> TTSModel: if net_g is not None: net_g = net_g.float() setattr(model, "_TTSModel__net_g", net_g) - _LOGGER.info("TTS network cast to float32") _LOGGER.info("Model loaded successfully") return model @@ -109,12 +125,22 @@ class GLaDOSEventHandler(AsyncEventHandler): model_dir: Path, device: str, *args, + half: bool = False, + preload: bool = False, **kwargs, ) -> None: super().__init__(*args, **kwargs) self.wyoming_info_event = wyoming_info.event() self.model_dir = model_dir self.device = device + self.half = half + + _optimize_gpu() + + if preload: + _LOGGER.info("Pre-loading model at startup...") + global _MODEL + _MODEL = _load_model(model_dir, device, half) async def handle_event(self, event: Event) -> bool: if Describe.is_type(event.type): @@ -152,9 +178,9 @@ class GLaDOSEventHandler(AsyncEventHandler): if _MODEL is None: _LOGGER.info("Loading GLaDOS model from %s on %s", self.model_dir, self.device) - _MODEL = _load_model(self.model_dir, self.device) + _MODEL = _load_model(self.model_dir, self.device, self.half) - _load_bert_for_language(language, self.device) + _load_bert_for_language(language, self.device, self.half) sr, audio = await asyncio.to_thread( _MODEL.infer,