diff --git a/cli/setup_mit_audio b/cli/setup_mit_audio index e5a1f23..507ebcd 100755 --- a/cli/setup_mit_audio +++ b/cli/setup_mit_audio @@ -25,9 +25,9 @@ fi mkdir -p "${DATA_DIR}/training_datasets/downloads" || : cd "${DATA_DIR}/training_datasets" -AUDIO_URL="https://mcdermottlab.mit.edu/Reverb/IRMAudio/Audio.zip" -AUDIO_ZIPFILE="MIT_RIR_Audio.zip" -AUDIO_ZIP="./downloads/${AUDIO_ZIPFILE}" +HF_RIR_REPO_ID="TaterTotterson/MIT_environmental_impulse_responses" +HF_RIR_API_URL="https://huggingface.co/api/datasets/${HF_RIR_REPO_ID}" +HF_RIR_SOURCE_KEY="hf_mit_environmental_impulse_responses" AUDIO_DIR="./mit_rirs" mkdir -p "${AUDIO_DIR}" || : AUDIO16K_DIR="./mit_rirs_16k" @@ -35,10 +35,58 @@ mkdir -p "${AUDIO16K_DIR}" || : AUDIO_FILECOUNT="./downloads/mit_rir_filecount" AUDIO_IN_GLOB="*.wav" -declare -A filecounts=( [${AUDIO_ZIPFILE}]=0 ) +declare -A filecounts=( [${HF_RIR_SOURCE_KEY}]=0 ) get_filecounts filecounts "${AUDIO_FILECOUNT}" -echo "===== Checking MIT_RIR =====" +echo "===== Checking MIT environmental RIRs =====" + +download_hf_mit_rirs() { + source ${DATA_DIR}/.venv/bin/activate + python - "${HF_RIR_REPO_ID}" "${HF_RIR_API_URL}" "${AUDIO_DIR}" <<-'EOF' +import json +import sys +import urllib.parse +import urllib.request +from pathlib import Path + +repo_id = sys.argv[1] +api_url = sys.argv[2] +audio_dir = Path(sys.argv[3]) +audio_dir.mkdir(parents=True, exist_ok=True) + +request = urllib.request.Request(api_url, headers={"User-Agent": "WakeWordTrainer/1.0"}) +with urllib.request.urlopen(request, timeout=30) as response: + metadata = json.loads(response.read().decode("utf-8")) + +files = sorted( + sibling.get("rfilename", "") + for sibling in metadata.get("siblings", []) + if str(sibling.get("rfilename", "")).startswith("16khz/") + and str(sibling.get("rfilename", "")).lower().endswith(".wav") +) +if not files: + raise SystemExit("Hugging Face MIT RIR dataset did not list any 16khz WAV files") + +downloaded = 0 +skipped = 0 +for rel in files: + target = audio_dir / rel + if target.exists() and target.stat().st_size > 0: + skipped += 1 + continue + target.parent.mkdir(parents=True, exist_ok=True) + encoded = urllib.parse.quote(rel, safe="/") + url = f"https://huggingface.co/datasets/{repo_id}/resolve/main/{encoded}" + with urllib.request.urlopen(url, timeout=60) as response: + target.write_bytes(response.read()) + if not target.exists() or target.stat().st_size == 0: + raise SystemExit(f"download failed for {rel}") + downloaded += 1 + +print(f" Hugging Face MIT environmental RIR download complete ({downloaded} downloaded, {skipped} reused)") +print(f" MIT environmental RIR files available: {len(files)}") +EOF +} converter() { source ${DATA_DIR}/.venv/bin/activate @@ -58,9 +106,9 @@ rir_out = Path(sys.argv[2]) waves = list(rir_in.rglob("*.wav")) try: - print(" MIT RIR normalizing to 16k…") + print(" MIT environmental RIR normalizing to 16k…") # Normalize to 16k mono - for p in tqdm(waves, desc=" MIT_RIR (resample 16k mono)"): + for p in tqdm(waves, desc=" MIT environmental RIR (resample 16k mono)"): outfile = Path(rir_out / p.name) if outfile.exists(): continue @@ -70,14 +118,14 @@ try: if sr != 16000: a, _ = librosa.load(p, sr=16000, mono=True) write_wav(outfile, a, 16000) - print(" MIT RIR normalization complete") + print(" MIT environmental RIR normalization complete") except Exception as e2: - print(f" MIT RIR fallback failed: {e2}") + print(f" MIT environmental RIR preparation failed: {e2}") raise EOF } -expected_filecount=${filecounts[${AUDIO_ZIPFILE}]} +expected_filecount=${filecounts[${HF_RIR_SOURCE_KEY}]} actual_filecount=$(find "${AUDIO16K_DIR}" -name '*.wav' 2>/dev/null | wc -l) || : write_filecount=false @@ -85,24 +133,16 @@ if [ "${actual_filecount}" -ne 0 ] && [ "${actual_filecount}" -eq "${expected_fi echo " Existing ${AUDIO16K_DIR} valid" else actual_filecount=$(find "${AUDIO_DIR}" -name "${AUDIO_IN_GLOB}" 2>/dev/null | wc -l) || : - if [ "${actual_filecount}" -eq 0 ] || [ "${actual_filecount}" -ne "${expected_filecount}" ] ; then - if [ ! -f "${AUDIO_ZIP}" ] ; then - echo " Downloading ${AUDIO_ZIPFILE}" - curl -sfL "${AUDIO_URL}" -o "${AUDIO_ZIP}" - fi - + if [ "${actual_filecount}" -eq 0 ] || [ "${expected_filecount}" -eq 0 ] || [ "${actual_filecount}" -ne "${expected_filecount}" ] ; then rm -rf "${AUDIO_DIR}" || : - echo " Unzipping ${AUDIO_ZIPFILE}" - unzip -u -q -d "${AUDIO_DIR}" "${AUDIO_ZIP}" - fi - if "${CLEANUP_ARCHIVES}" && [ -f "${AUDIO_ZIP}" ] ; then - echo " Cleaning up ${AUDIO_ZIPFILE}" - rm -rf "${AUDIO_ZIP}" + mkdir -p "${AUDIO_DIR}" || : + echo " Downloading MIT environmental impulse responses from Hugging Face mirror" + download_hf_mit_rirs fi converter actual_filecount=$(find "${AUDIO16K_DIR}" -name "*.wav" 2>/dev/null | wc -l) || : - filecounts[${AUDIO_ZIPFILE}]="${actual_filecount}" + filecounts[${HF_RIR_SOURCE_KEY}]="${actual_filecount}" write_filecount=true fi @@ -110,15 +150,10 @@ if ${write_filecount} ; then write_filecounts filecounts "${AUDIO_FILECOUNT}" fi -if "${CLEANUP_ARCHIVES}" && [ -f "${AUDIO_ZIP}" ] ; then - echo " Cleaning up ${AUDIO_ZIPFILE}" - rm -rf "${AUDIO_ZIP}" -fi - if "${CLEANUP_INTERMEDIATE_FILES}" && [ -d "${AUDIO_DIR}" ]; then echo " Cleaning up ${AUDIO_DIR}" rm -rf "${AUDIO_DIR}" fi -echo " MIT_RIR complete" +echo " MIT environmental RIRs complete" exit 0