Show MIT RIR download progress

cli/setup_mit_audio

@@ -25,9 +25,9 @@ fi
 mkdir -p "${DATA_DIR}/training_datasets/downloads" || :
 cd "${DATA_DIR}/training_datasets"
 
-AUDIO_URL="https://mcdermottlab.mit.edu/Reverb/IRMAudio/Audio.zip"
+HF_RIR_REPO_ID="TaterTotterson/MIT_environmental_impulse_responses"
-AUDIO_ZIPFILE="MIT_RIR_Audio.zip"
+HF_RIR_API_URL="https://huggingface.co/api/datasets/${HF_RIR_REPO_ID}"
-AUDIO_ZIP="./downloads/${AUDIO_ZIPFILE}"
+HF_RIR_SOURCE_KEY="hf_mit_environmental_impulse_responses"
 AUDIO_DIR="./mit_rirs"
 mkdir -p "${AUDIO_DIR}" || :
 AUDIO16K_DIR="./mit_rirs_16k"
@@ -35,10 +35,92 @@ mkdir -p "${AUDIO16K_DIR}" || :
 AUDIO_FILECOUNT="./downloads/mit_rir_filecount"
 AUDIO_IN_GLOB="*.wav"
 
-declare -A filecounts=( [${AUDIO_ZIPFILE}]=0 )
+declare -A filecounts=( [${HF_RIR_SOURCE_KEY}]=0 )
 get_filecounts filecounts "${AUDIO_FILECOUNT}"
 
-echo "===== Checking MIT_RIR ====="
+echo "===== Checking MIT environmental RIRs ====="
+
+download_hf_mit_rirs() {
+    source ${DATA_DIR}/.venv/bin/activate
+    python - "${HF_RIR_REPO_ID}" "${HF_RIR_API_URL}" "${AUDIO_DIR}" <<-'EOF'
+import json
+import sys
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+repo_id = sys.argv[1]
+api_url = sys.argv[2]
+audio_dir = Path(sys.argv[3])
+audio_dir.mkdir(parents=True, exist_ok=True)
+
+request = urllib.request.Request(api_url, headers={"User-Agent": "WakeWordTrainer/1.0"})
+with urllib.request.urlopen(request, timeout=30) as response:
+    metadata = json.loads(response.read().decode("utf-8"))
+
+files = sorted(
+    sibling.get("rfilename", "")
+    for sibling in metadata.get("siblings", [])
+    if str(sibling.get("rfilename", "")).startswith("16khz/")
+    and str(sibling.get("rfilename", "")).lower().endswith(".wav")
+)
+if not files:
+    raise SystemExit("Hugging Face MIT RIR dataset did not list any 16khz WAV files")
+
+print(f"   Found {len(files)} MIT environmental RIR files on Hugging Face mirror", flush=True)
+downloaded = 0
+skipped = 0
+
+def download_file(url: str, target: Path, rel: str):
+    tmp = target.with_suffix(target.suffix + ".incomplete")
+    for attempt in range(1, 4):
+        try:
+            if tmp.exists():
+                tmp.unlink()
+            with urllib.request.urlopen(url, timeout=30) as response:
+                with tmp.open("wb") as out:
+                    while True:
+                        chunk = response.read(1024 * 64)
+                        if not chunk:
+                            break
+                        out.write(chunk)
+            if not tmp.exists() or tmp.stat().st_size == 0:
+                raise RuntimeError("empty download")
+            tmp.replace(target)
+            return
+        except Exception as exc:
+            if tmp.exists():
+                tmp.unlink()
+            if attempt == 3:
+                raise RuntimeError(f"download failed for {rel}: {exc}") from exc
+            print(f"   Retry {attempt}/2 for {rel}: {exc}", flush=True)
+            time.sleep(2 * attempt)
+
+total = len(files)
+for idx, rel in enumerate(files, start=1):
+    target = audio_dir / rel
+    if target.exists() and target.stat().st_size > 0:
+        skipped += 1
+        if idx == 1 or idx % 25 == 0 or idx == total:
+            print(f"   MIT RIR download progress: {idx}/{total} files ({downloaded} downloaded, {skipped} reused)", flush=True)
+        continue
+    target.parent.mkdir(parents=True, exist_ok=True)
+    encoded = urllib.parse.quote(rel, safe="/")
+    url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{encoded}"
+    if idx == 1 or idx % 25 == 0 or idx == total:
+        print(f"   Downloading MIT RIR {idx}/{total}: {rel}", flush=True)
+    download_file(url, target, rel)
+    if not target.exists() or target.stat().st_size == 0:
+        raise SystemExit(f"download failed for {rel}")
+    downloaded += 1
+    if idx == 1 or idx % 25 == 0 or idx == total:
+        print(f"   MIT RIR download progress: {idx}/{total} files ({downloaded} downloaded, {skipped} reused)", flush=True)
+
+print(f"   Hugging Face MIT environmental RIR download complete ({downloaded} downloaded, {skipped} reused)", flush=True)
+print(f"   MIT environmental RIR files available: {len(files)}", flush=True)
+EOF
+}
 
 converter() {
     source ${DATA_DIR}/.venv/bin/activate
@@ -58,9 +140,9 @@ rir_out = Path(sys.argv[2])
 
 waves = list(rir_in.rglob("*.wav"))
 try:
-    print("   MIT RIR normalizing to 16k…")
+    print("   MIT environmental RIR normalizing to 16k…")
     # Normalize to 16k mono
-    for p in tqdm(waves, desc="   MIT_RIR (resample 16k mono)"):
+    for p in tqdm(waves, desc="   MIT environmental RIR (resample 16k mono)"):
         outfile = Path(rir_out / p.name)
         if outfile.exists():
             continue
@@ -70,14 +152,14 @@ try:
         if sr != 16000:
             a, _ = librosa.load(p, sr=16000, mono=True)
         write_wav(outfile, a, 16000)
-    print("   MIT RIR normalization complete")
+    print("   MIT environmental RIR normalization complete")
 except Exception as e2:
-    print(f"   MIT RIR fallback failed: {e2}")
+    print(f"   MIT environmental RIR preparation failed: {e2}")
     raise
 EOF
 }
 
-expected_filecount=${filecounts[${AUDIO_ZIPFILE}]}
+expected_filecount=${filecounts[${HF_RIR_SOURCE_KEY}]}
 actual_filecount=$(find "${AUDIO16K_DIR}" -name '*.wav' 2>/dev/null | wc -l) || :
 write_filecount=false
 
@@ -85,24 +167,16 @@ if [ "${actual_filecount}" -ne 0 ] && [ "${actual_filecount}" -eq "${expected_fi
     echo "   Existing ${AUDIO16K_DIR} valid"
 else
     actual_filecount=$(find "${AUDIO_DIR}" -name "${AUDIO_IN_GLOB}" 2>/dev/null | wc -l) || :
-    if [ "${actual_filecount}" -eq 0 ] || [ "${actual_filecount}" -ne "${expected_filecount}" ] ; then
+    if [ "${actual_filecount}" -eq 0 ] || [ "${expected_filecount}" -eq 0 ] || [ "${actual_filecount}" -ne "${expected_filecount}" ] ; then
-        if [ ! -f "${AUDIO_ZIP}" ]  ; then
-            echo "   Downloading ${AUDIO_ZIPFILE}"
-            curl -sfL "${AUDIO_URL}" -o "${AUDIO_ZIP}"
-        fi
-
         rm -rf "${AUDIO_DIR}" || :
-        echo "   Unzipping ${AUDIO_ZIPFILE}"
+        mkdir -p "${AUDIO_DIR}" || :
-        unzip -u -q -d "${AUDIO_DIR}" "${AUDIO_ZIP}"
+        echo "   Downloading MIT environmental impulse responses from Hugging Face mirror"
-    fi
+        download_hf_mit_rirs
-    if "${CLEANUP_ARCHIVES}" && [ -f "${AUDIO_ZIP}" ] ; then
-        echo "   Cleaning up ${AUDIO_ZIPFILE}"
-        rm -rf "${AUDIO_ZIP}"
     fi
 
     converter
     actual_filecount=$(find "${AUDIO16K_DIR}" -name "*.wav" 2>/dev/null | wc -l) || :
-    filecounts[${AUDIO_ZIPFILE}]="${actual_filecount}"
+    filecounts[${HF_RIR_SOURCE_KEY}]="${actual_filecount}"
     write_filecount=true
 fi
 
@@ -110,15 +184,10 @@ if ${write_filecount} ; then
     write_filecounts filecounts "${AUDIO_FILECOUNT}"
 fi
 
-if "${CLEANUP_ARCHIVES}" && [ -f "${AUDIO_ZIP}" ] ; then
-    echo "   Cleaning up ${AUDIO_ZIPFILE}"
-    rm -rf "${AUDIO_ZIP}"
-fi
-
 if "${CLEANUP_INTERMEDIATE_FILES}" && [ -d "${AUDIO_DIR}" ]; then
     echo "   Cleaning up ${AUDIO_DIR}"
     rm -rf "${AUDIO_DIR}"
 fi
 
-echo "   MIT_RIR complete"
+echo "   MIT environmental RIRs complete"
 exit 0