Add browser USB firmware flashing

README.md

@@ -187,11 +187,12 @@ After those assets are prepared, later runs reuse the local copies unless the mo
 
 The `Firmware` tab flashes prebuilt Tater firmware for supported ESPHome satellites.
 
-- Downloads the latest prebuilt firmware manifest and OTA image from `TaterTotterson/microWakeWords`.
+- Downloads the latest prebuilt firmware manifest plus OTA and USB factory images from `TaterTotterson/microWakeWords`.
 - Verifies downloaded images by size and SHA before upload.
 - Auto-detects ESPHome devices with mDNS when the container is running with host networking.
 - Allows manual IP or hostname entry if discovery does not find the device.
 - Saves the selected OTA target for each firmware family.
+- Flashes the prebuilt factory image over Browser USB for first installs or recovery when opened in Chrome or Edge.
 - Lists locally trained wake words from `/data/trained_wake_words/` for live model switching.
 - Streams download, verification, and OTA upload progress in a colorized firmware console.
 

static/index.html

@@ -503,6 +503,12 @@
       justify-content: flex-end;
     }
 
+    .usbFlashHint {
+      color: var(--muted);
+      font-size: 12px;
+      max-width: 420px;
+    }
+
     .studioHero {
       position: relative;
       overflow: hidden;
@@ -1524,8 +1530,7 @@
             <div class="firmwareSteps" aria-label="Firmware flashing steps">
               <span class="firmwareStepChip"><b>1</b> Pick firmware</span>
               <span class="firmwareStepChip"><b>2</b> Select target</span>
-              <span class="firmwareStepChip"><b>3</b> Verify image</span>
-              <span class="firmwareStepChip"><b>4</b> Flash OTA</span>
+              <span class="firmwareStepChip"><b>3</b> Flash OTA or USB</span>
             </div>
           </div>
           <span id="firmwareStatus" class="pill">Flasher idle</span>
@@ -1601,35 +1606,24 @@
         </section>
       </div>
 
-      <section class="card firmwarePanel stack">
-        <div class="firmwarePanelHeader">
-          <div class="firmwarePanelTitle">
-            <span class="firmwareStepBadge">3</span>
-            <div>
-              <h3>Prebuilt Image</h3>
-              <p>The trainer downloads the latest verified OTA image and checks its size and SHA before flashing.</p>
-            </div>
-          </div>
-          <button id="saveFirmwareSettingsBtn" type="button">Save target</button>
-        </div>
-        <div id="firmwareFields" class="firmwareFields stack">
-          <div class="emptyState">Firmware image details will appear here.</div>
-        </div>
-      </section>
+      <div id="firmwareFields" class="firmwareFields stack" hidden></div>
 
       <section class="card firmwareActionsPanel">
         <div class="firmwarePanelTitle">
-          <span class="firmwareStepBadge">4</span>
+          <span class="firmwareStepBadge">3</span>
           <div>
-            <h3>Flash OTA</h3>
-            <p>The console shows download, verification, OTA upload, and reboot progress.</p>
+            <h3>Flash Firmware</h3>
+            <p>The console shows download, verification, OTA upload, USB flash, and reboot progress.</p>
           </div>
         </div>
         <div class="row firmwareActions">
-          <button id="flashFirmwareBtn" class="primary" type="button" disabled>Flash prebuilt firmware</button>
+          <button id="flashFirmwareBtn" class="primary" type="button" disabled>Flash OTA Update</button>
+          <button id="usbFirmwareBtn" type="button" disabled>Browser USB Flash</button>
+          <button id="saveFirmwareSettingsBtn" type="button">Save target</button>
           <button id="cleanFirmwareBtn" type="button">Clear downloaded images</button>
           <button id="openFirmwareConsoleBtn" type="button">Open firmware console</button>
         </div>
+        <div class="usbFlashHint">Use Browser USB Flash for new satellites, recovery, or devices older than Tater firmware 3.0.3. Chrome or Edge must be used on localhost or HTTPS.</div>
       </section>
     </div>
   </div>
@@ -1710,6 +1704,7 @@
     captured: { items: [], captured_count: 0, negative_count: 0, personal_count: 0 },
     samples: { personal: [], negative: [], personal_count: 0, negative_count: 0, activeBucket: "personal", pages: { personal: 0, negative: 0 } },
     firmware: { devices: [], templates: [], wakeWords: [], flashing: null, logLines: [], activeTemplateKey: "" },
+    browserUsb: { ports: {}, esptoolJsModule: null, esptoolJsLoadPromise: null },
     uploadBusy: false,
     reviewBusy: false,
     firmwareBusy: false,
@@ -2362,6 +2357,23 @@
     $("firmwareLogStatus").textContent = String(text || "Waiting for firmware output...").trim() || "Waiting for firmware output...";
   }
 
+  function appendFirmwareLog(line, statusText = "") {
+    const text = String(line || "").trim();
+    if (!text) return;
+    uiState.firmware.logLines = [...(uiState.firmware.logLines || []), text].slice(-500);
+    renderFirmwareLogLines(uiState.firmware.logLines, true);
+    if (statusText) {
+      setFirmwareLogStatus(statusText);
+    }
+  }
+
+  function appendFirmwareLogs(lines, statusText = "") {
+    (Array.isArray(lines) ? lines : []).forEach((line) => appendFirmwareLog(line));
+    if (statusText) {
+      setFirmwareLogStatus(statusText);
+    }
+  }
+
   function openFirmwareConsole(wobble = true, text = null, statusText = "") {
     const template = selectedFirmwareTemplate();
     const host = ($("firmwareHost").value || "").trim();
@@ -2409,6 +2421,221 @@
     });
   }
 
+  function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, Math.max(0, Number(ms) || 0)));
+  }
+
+  function browserUsbPortLabel(port) {
+    if (!port || typeof port.getInfo !== "function") {
+      return "USB serial device";
+    }
+    const info = port.getInfo() || {};
+    const vendor = Number(info.usbVendorId || 0);
+    const product = Number(info.usbProductId || 0);
+    if (!vendor && !product) {
+      return "USB serial device";
+    }
+    const hex = (value) => value.toString(16).padStart(4, "0");
+    return `USB ${hex(vendor)}:${hex(product)}`;
+  }
+
+  async function browserUsbSelectPort(selector = "") {
+    if (!window.isSecureContext || !navigator.serial) {
+      throw new Error("Browser USB flash needs Chrome or Edge on HTTPS or localhost. In the macOS app, use Open in Browser and choose Chrome or Edge.");
+    }
+    const port = await navigator.serial.requestPort();
+    const key = String(selector || "default").trim() || "default";
+    uiState.browserUsb.ports[key] = port;
+    return port;
+  }
+
+  function normalizeEsptoolJsModule(module) {
+    if (module?.ESPLoader && module?.Transport) {
+      return module;
+    }
+    if (module?.default?.ESPLoader && module?.default?.Transport) {
+      return module.default;
+    }
+    return null;
+  }
+
+  async function ensureEsptoolJsLoaded() {
+    if (uiState.browserUsb.esptoolJsModule) {
+      return uiState.browserUsb.esptoolJsModule;
+    }
+    if (uiState.browserUsb.esptoolJsLoadPromise instanceof Promise) {
+      return uiState.browserUsb.esptoolJsLoadPromise;
+    }
+    const moduleUrls = [
+      "https://esm.sh/esptool-js@0.5.7?bundle",
+      "https://unpkg.com/esptool-js@0.5.7/bundle.js",
+      "https://cdn.jsdelivr.net/npm/esptool-js@0.5.7/bundle.js",
+    ];
+    uiState.browserUsb.esptoolJsLoadPromise = (async () => {
+      const errors = [];
+      for (const moduleUrl of moduleUrls) {
+        try {
+          const module = normalizeEsptoolJsModule(await import(moduleUrl));
+          if (module) {
+            uiState.browserUsb.esptoolJsModule = module;
+            return module;
+          }
+          errors.push(`${moduleUrl}: missing ESPLoader/Transport exports`);
+        } catch (error) {
+          errors.push(`${moduleUrl}: ${String(error?.message || error || "load failed")}`);
+        }
+      }
+      throw new Error(`Could not load esptool-js. ${errors.join(" | ")}`);
+    })();
+    return uiState.browserUsb.esptoolJsLoadPromise;
+  }
+
+  async function fetchFirmwareBinary(binaryUrl) {
+    const response = await fetch(binaryUrl, { credentials: "same-origin" });
+    if (!response.ok) {
+      throw new Error(`Firmware download failed: HTTP ${response.status}`);
+    }
+    return new Uint8Array(await response.arrayBuffer());
+  }
+
+  function uint8ArrayToBinaryString(bytes) {
+    const chunks = [];
+    const chunkSize = 0x8000;
+    for (let offset = 0; offset < bytes.length; offset += chunkSize) {
+      chunks.push(String.fromCharCode(...bytes.subarray(offset, offset + chunkSize)));
+    }
+    return chunks.join("");
+  }
+
+  async function browserUsbSetSignals(port, signals, timeoutMs = 1200) {
+    if (!port || typeof port.setSignals !== "function") {
+      return false;
+    }
+    let timeoutId = 0;
+    try {
+      await Promise.race([
+        port.setSignals(signals),
+        new Promise((_resolve, reject) => {
+          timeoutId = setTimeout(() => reject(new Error("Timed out setting USB serial control signals.")), Math.max(1, Number(timeoutMs) || 1));
+        }),
+      ]);
+      return true;
+    } finally {
+      if (timeoutId) {
+        clearTimeout(timeoutId);
+      }
+    }
+  }
+
+  async function browserUsbHardResetAfterFlash(transport, loader, port) {
+    try {
+      appendFirmwareLog("Resetting device after USB flash.");
+      if (transport && typeof transport.setRTS === "function") {
+        await transport.setRTS(true);
+      } else {
+        await browserUsbSetSignals(port, { dataTerminalReady: false, requestToSend: true, break: false });
+      }
+      await sleep(100);
+      if (loader && typeof loader.after === "function") {
+        await loader.after();
+      }
+      await sleep(1000);
+      return true;
+    } catch (error) {
+      appendFirmwareLog(`USB reset warning: ${String(error?.message || error)}`);
+      return false;
+    }
+  }
+
+  async function flashBrowserUsbPort(port, artifact) {
+    const module = await ensureEsptoolJsLoaded();
+    const { ESPLoader, Transport } = module;
+    if (!ESPLoader || !Transport) {
+      throw new Error("esptool-js did not expose ESPLoader/Transport.");
+    }
+    const binaryUrl = String(artifact?.binary_url || "").trim();
+    if (!binaryUrl) {
+      throw new Error("Browser flash binary URL is missing.");
+    }
+
+    const transport = new Transport(port);
+    const terminal = {
+      clean() {},
+      writeLine(data) {
+        const text = String(data || "").trim();
+        if (text) appendFirmwareLog(text);
+      },
+      write(data) {
+        const text = String(data || "").trim();
+        if (text) appendFirmwareLog(text);
+      },
+    };
+    const loader = new ESPLoader({
+      transport,
+      baudrate: 115200,
+      terminal,
+      debugLogging: false,
+    });
+
+    let lastProgress = -1;
+    try {
+      appendFirmwareLog(`Connecting to ${browserUsbPortLabel(port)}...`, "Connecting to selected USB device...");
+      const chipName = await loader.main();
+      appendFirmwareLog(`Connected to ${chipName || "ESP device"}.`);
+
+      setFirmwareLogStatus("Downloading prepared factory firmware...");
+      const firmwareData = await fetchFirmwareBinary(binaryUrl);
+      appendFirmwareLog(`Firmware downloaded (${firmwareData.byteLength} bytes).`);
+
+      const eraseAll = artifact?.erase_all !== false;
+      if (eraseAll) {
+        appendFirmwareLog("Erasing flash and writing factory image at offset 0x0.");
+      } else {
+        appendFirmwareLog("Writing factory image at offset 0x0.");
+      }
+      const flashOptions = {
+        fileArray: [{ data: firmwareData, address: 0 }],
+        flashMode: String(artifact?.flash_mode || "dio"),
+        flashFreq: String(artifact?.flash_freq || "40m"),
+        flashSize: String(artifact?.flash_size || "4MB"),
+        eraseAll,
+        compress: true,
+        reportProgress: (_fileIndex, written, total) => {
+          const pct = total > 0 ? Math.floor((written / total) * 100) : 0;
+          if (pct >= lastProgress + 5 || pct === 100) {
+            lastProgress = pct;
+            appendFirmwareLog(`USB flash progress: ${pct}% (${written}/${total} bytes)`, `Flashing over USB... ${pct}%`);
+          }
+        },
+      };
+      try {
+        await loader.writeFlash(flashOptions);
+      } catch (error) {
+        const message = String(error?.message || error || "");
+        if (!message.includes("charCodeAt")) {
+          throw error;
+        }
+        appendFirmwareLog("Retrying flash with binary-string firmware format.");
+        await loader.writeFlash({
+          ...flashOptions,
+          eraseAll: false,
+          fileArray: [{ data: uint8ArrayToBinaryString(firmwareData), address: 0 }],
+        });
+      }
+      appendFirmwareLog("USB flash complete.");
+      const resetWorked = await browserUsbHardResetAfterFlash(transport, loader, port);
+      if (!resetWorked) {
+        appendFirmwareLog("If the device does not restart, unplug and reconnect USB once to leave bootloader mode.");
+      }
+    } finally {
+      try {
+        await transport.disconnect();
+      } catch (_error) {
+        // Ignore disconnect cleanup failures.
+      }
+    }
+  }
+
   function closeConsole() {
     $("consoleOverlay").classList.remove("open");
     $("consoleOverlay").setAttribute("aria-hidden", "true");
@@ -3033,6 +3260,66 @@
     }
   }
 
+  async function startUsbFirmwareFlash() {
+    const template = selectedFirmwareTemplate();
+    if (!template) {
+      alert("Choose a firmware family first.");
+      return;
+    }
+    const factory = template.prebuilt_firmware?.artifacts?.factory || {};
+    if (!factory.path) {
+      alert("No prebuilt USB factory image is available for this firmware family.");
+      return;
+    }
+    if (!window.isSecureContext || !navigator.serial) {
+      alert("Browser USB flash needs Chrome or Edge on HTTPS or localhost. In the macOS app, use Open in Browser and choose Chrome or Edge.");
+      return;
+    }
+    const ok = confirm(`USB flash the prebuilt ${template.label || template.value} factory firmware?\n\nThis is for new satellites, recovery, or devices older than Tater firmware 3.0.3. It erases flash and writes the factory image over USB.`);
+    if (!ok) return;
+
+    let port;
+    try {
+      port = await browserUsbSelectPort(template.value);
+    } catch (error) {
+      alert("USB device selection failed: " + error.message);
+      return;
+    }
+
+    uiState.firmwareBusy = true;
+    uiState.firmware.logLines = [
+      "===== Browser USB Firmware Flash Console =====",
+      `Firmware: ${template.label || template.value}`,
+      `USB device: ${browserUsbPortLabel(port)}`,
+      "Preparing verified factory image...",
+    ];
+    setPill($("firmwareStatus"), "Preparing USB flash...", "warn");
+    openFirmwareConsole(true, uiState.firmware.logLines.join("\n"), "Preparing browser USB flash...");
+    syncButtons();
+    await waitForPaint();
+
+    try {
+      const artifact = await api("/api/firmware/browser_flash", {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+          template_key: template.value,
+        }),
+      });
+      appendFirmwareLogs(artifact.entries || [], "Factory image ready. Starting USB flash...");
+      await flashBrowserUsbPort(port, artifact);
+      appendFirmwareLog("Browser USB flash finished.", "Browser USB flash finished.");
+      setPill($("firmwareStatus"), "USB flash finished", "ok");
+    } catch (error) {
+      appendFirmwareLog(`Browser USB flash failed: ${String(error.message || error)}`, "Browser USB flash failed.");
+      setPill($("firmwareStatus"), "USB flash failed", "err");
+      alert("Browser USB flash failed: " + error.message);
+    } finally {
+      uiState.firmwareBusy = false;
+      syncButtons();
+    }
+  }
+
   async function pollFirmwareFlash(sessionId) {
     if (uiState.firmwarePoller || !sessionId) return;
     uiState.firmwarePoller = true;
@@ -3123,6 +3410,7 @@
     $("cleanFirmwareBtn").disabled = uiState.firmwareBusy;
     $("openFirmwareConsoleBtn").disabled = false;
     $("flashFirmwareBtn").disabled = uiState.firmwareBusy || !firmwareHost || !firmwareTemplate || !firmwareAvailable;
+    $("usbFirmwareBtn").disabled = uiState.firmwareBusy || !firmwareTemplate || !firmwareSelection?.prebuilt_firmware?.artifacts?.factory?.path;
   }
 
   function refreshSessionUI(session) {
@@ -3491,6 +3779,7 @@
     }
   });
   $("flashFirmwareBtn").addEventListener("click", startFirmwareFlash);
+  $("usbFirmwareBtn").addEventListener("click", startUsbFirmwareFlash);
   $("saveFirmwareSettingsBtn").addEventListener("click", async () => {
     const host = ($("firmwareHost").value || "").trim();
     const template = ($("firmwareTemplate").value || "").trim();

trainer_server.py

@@ -94,6 +94,7 @@ FIRMWARE_OTA_BLOCK_SIZE = int(os.environ.get("FIRMWARE_OTA_BLOCK_SIZE", "8192"))
 FIRMWARE_PROFILE_FILE = Path(
     os.environ.get("FIRMWARE_PROFILE_FILE", str(FIRMWARE_CACHE_DIR / "profiles.json"))
 ).resolve()
+FIRMWARE_WEB_FLASH_DIR = FIRMWARE_CACHE_DIR / "web_flash"
 WAKE_SOUND_MANIFEST_PATHS = ("wake_sound_manifest.json", "wake-sound-manifest.json")
 WAKE_SOUND_CATALOG_CACHE: Dict[str, Any] = {"ts": 0.0, "payload": {}}
 WAKE_SOUND_CATALOG_LOCK = threading.Lock()
@@ -1683,6 +1684,57 @@ def _download_prebuilt_firmware_binary(
     return {"path": target_path, "artifact": artifact, "url": url, "cached": False}
 
 
+def _browser_flash_artifact_id(template_key: Any) -> str:
+    return "_".join(
+        part
+        for part in [
+            _sanitize_token(template_key),
+            str(int(time.time())),
+            uuid.uuid4().hex[:8],
+        ]
+        if part
+    )
+
+
+def _create_browser_flash_artifact(template_key: Any, prebuilt: Dict[str, Any], binary_path: Path) -> Dict[str, Any]:
+    artifact_id = _browser_flash_artifact_id(template_key)
+    artifact_dir = FIRMWARE_WEB_FLASH_DIR / artifact_id
+    artifact_dir.mkdir(parents=True, exist_ok=True)
+
+    target_binary_name = "firmware.bin"
+    target_binary_path = artifact_dir / target_binary_name
+    shutil.copy2(binary_path, target_binary_path)
+    return {
+        "artifact_id": artifact_id,
+        "binary_url": f"/api/firmware/browser_flash/{artifact_id}/{target_binary_name}",
+        "binary_name": target_binary_name,
+        "template_key": _text(template_key),
+        "firmware_version": _text(prebuilt.get("version")),
+        "source_binary": str(binary_path),
+        "binary_size": int(target_binary_path.stat().st_size),
+        "erase_all": True,
+        "flash_size": "4MB",
+        "flash_mode": "dio",
+        "flash_freq": "40m",
+    }
+
+
+def _browser_flash_artifact_path(artifact_id: str, relative_path: str) -> Path:
+    artifact = _sanitize_token(artifact_id)
+    if not artifact:
+        raise KeyError("Browser flash artifact is missing.")
+    rel = Path(_text(relative_path))
+    if rel.is_absolute() or any(part in {"", ".", ".."} for part in rel.parts):
+        raise KeyError("Browser flash artifact path is invalid.")
+    root = (FIRMWARE_WEB_FLASH_DIR / artifact).resolve()
+    target = (root / rel).resolve()
+    if root not in target.parents and target != root:
+        raise KeyError("Browser flash artifact path is invalid.")
+    if not target.is_file():
+        raise KeyError("Browser flash artifact file was not found.")
+    return target
+
+
 class _NativeOTAError(RuntimeError):
     pass
 
@@ -3346,6 +3398,50 @@ def firmware_build_flash(payload: Dict[str, Any]):
     return _firmware_session_payload(session_id)
 
 
+@app.post("/api/firmware/browser_flash")
+def firmware_browser_flash(payload: Dict[str, Any]):
+    body = payload if isinstance(payload, dict) else {}
+    try:
+        template_key = str(body.get("template_key") or "").strip()
+        template_spec = _firmware_template_spec(template_key)
+        prebuilt = _prebuilt_firmware_info(template_key, force_refresh=True)
+        if not bool(prebuilt.get("available")):
+            raise RuntimeError(_text(prebuilt.get("error")) or "No prebuilt firmware image is available for this firmware target.")
+        _prebuilt_artifact_meta(prebuilt, "factory")
+        binary = _download_prebuilt_firmware_binary(template_key, prebuilt, "factory", force_refresh=True)
+        artifact = _create_browser_flash_artifact(template_key, prebuilt, Path(binary["path"]))
+        template_label = str(template_spec.get("label") or template_key)
+        cached_text = "cached" if bool(binary.get("cached")) else "downloaded"
+        return {
+            "ok": True,
+            "template_key": template_key,
+            "template_label": template_label,
+            "firmware_version": _text(prebuilt.get("version")),
+            "message": f"Prepared prebuilt USB firmware for {template_label}.",
+            "entries": [
+                f"Using prebuilt {template_label} factory firmware {_text(prebuilt.get('version')) or 'latest'}.",
+                f"Factory image {cached_text}: {Path(binary['path']).name}",
+                f"Browser flash binary ready: {artifact['binary_name']} ({artifact['binary_size']} bytes)",
+            ],
+            **artifact,
+        }
+    except Exception as e:
+        return JSONResponse({"ok": False, "error": str(e)}, status_code=400)
+
+
+@app.get("/api/firmware/browser_flash/{artifact_id}/{filename}")
+def firmware_browser_flash_binary(artifact_id: str, filename: str):
+    try:
+        target = _browser_flash_artifact_path(artifact_id, filename)
+    except KeyError as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=404)
+    return FileResponse(
+        str(target),
+        media_type="application/octet-stream",
+        filename=target.name,
+    )
+
+
 @app.post("/api/firmware/clean")
 def firmware_clean():
     active = []
@@ -3357,7 +3453,7 @@ def firmware_clean():
         return JSONResponse({"ok": False, "error": f"Wait for active firmware session(s) to finish: {', '.join(active[:3])}."}, status_code=400)
 
     removed = []
-    for child in ("prebuilt_firmware", "uploads"):
+    for child in ("prebuilt_firmware", "uploads", "web_flash"):
         path = FIRMWARE_CACHE_DIR / child
         if path.exists():
             shutil.rmtree(path, ignore_errors=True)