From 3dd305e5600453f4b833fe218d5f83e645225fac Mon Sep 17 00:00:00 2001
From: MasterPhooey <hatted-limited0r@icloud.com>
Date: Wed, 31 Dec 2025 08:24:59 -0600
Subject: [PATCH] training live streaming

---
 microWakeWord_training_notebook.ipynb | 28 +++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/microWakeWord_training_notebook.ipynb b/microWakeWord_training_notebook.ipynb
index 210b643..118128d 100644
--- a/microWakeWord_training_notebook.ipynb
+++ b/microWakeWord_training_notebook.ipynb
@@ -853,6 +853,7 @@
    "outputs": [],
    "source": [
     "# Train + export with GPU first, then automatic CPU fallback on GPU/VRAM errors\n",
+    "# (LIVE streaming output + full log capture for error detection)\n",
     "import os, sys, subprocess, textwrap\n",
     "\n",
     "# ---- Common TF env (applies to BOTH attempts) ----\n",
@@ -903,20 +904,39 @@
     "    \"failed call to cuinit\",\n",
     ")\n",
     "\n",
-    "def run_training(label: str, extra_env: dict) -> subprocess.CompletedProcess:\n",
+    "class RunResult:\n",
+    "    def __init__(self, returncode: int, stdout: str):\n",
+    "        self.returncode = returncode\n",
+    "        self.stdout = stdout\n",
+    "\n",
+    "def run_training(label: str, extra_env: dict) -> RunResult:\n",
     "    env = base_env.copy()\n",
     "    env.update(extra_env or {})\n",
+    "\n",
     "    print(f\"\\n🚀 {label}\")\n",
     "    print(\"→\", \" \".join([sys.executable] + train_args))\n",
-    "    cp = subprocess.run(\n",
+    "\n",
+    "    proc = subprocess.Popen(\n",
     "        [sys.executable] + train_args,\n",
     "        env=env,\n",
     "        text=True,\n",
     "        stdout=subprocess.PIPE,\n",
     "        stderr=subprocess.STDOUT,\n",
+    "        bufsize=1,                    # line-buffered (best effort)\n",
+    "        universal_newlines=True,\n",
     "    )\n",
-    "    print(cp.stdout)\n",
-    "    return cp\n",
+    "\n",
+    "    full_log = []\n",
+    "    try:\n",
+    "        # Stream lines live AND capture them for OOM detection / error messages\n",
+    "        assert proc.stdout is not None\n",
+    "        for line in proc.stdout:\n",
+    "            print(line, end=\"\")\n",
+    "            full_log.append(line)\n",
+    "    finally:\n",
+    "        returncode = proc.wait()\n",
+    "\n",
+    "    return RunResult(returncode, \"\".join(full_log))\n",
     "\n",
     "# Attempt 1: GPU (normal visibility)\n",
     "cp = run_training(\n",