diff --git a/configs/katchi.yaml b/configs/katchi.yaml new file mode 100644 index 0000000..a70a1d6 --- /dev/null +++ b/configs/katchi.yaml @@ -0,0 +1,842 @@ +################################### +# _ __ _ _ _ # +# | |/ / | | | | | | # +# | ' / ___ | |__ ___ | | __| | # +# | < / _ \| '_ \ / _ \| |/ _` | # +# | . \ (_) | |_) | (_) | | (_| | # +# |_|\_\___/|_.__/ \___/|_|\__,_| # +################################### +# Voice assistant based on +# ESP32-S3-Touch-AMOLED-1.75 +# Incorporates the following: +# - Display -- CO5300 +# - Microphone -- ES7210 +# - Speaker -- ES8311 +################################## + +esphome: + name: kobold + friendly_name: Kobold + +esp32: + board: esp32-s3-devkitc-1 + flash_size: 16MB + cpu_frequency: 240MHZ + framework: + type: esp-idf + +################################### +# _____ __ _ # +# / ____| / _(_) # +# | | ___ _ __ | |_ _ __ _ # +# | | / _ \| '_ \| _| |/ _` | # +# | |___| (_) | | | | | | | (_| | # +# \_____\___/|_| |_|_| |_|\__, | # +# __/ | # +# |___/ # +#Config############################ + +substitutions: + voice_assist_idle_phase_id: "1" + voice_assist_listening_phase_id: "2" + voice_assist_thinking_phase_id: "3" + voice_assist_replying_phase_id: "4" + voice_assist_not_ready_phase_id: "10" + voice_assist_error_phase_id: "11" + voice_assist_muted_phase_id: "12" + voice_assist_timer_finished_phase_id: "20" + + i2s_bps_spk: 16bit + i2s_bps_mic: 16bit + i2s_sample_rate_spk: 44100 + i2s_sample_rate_mic: 16000 + + loading_illustration_file: kobold/loading.png + listening_illustration_file: kobold/listening.png + thinking_illustration_file: kobold/thinking.png + replying_illustration_file: kobold/responding.png + error_illustration_file: kobold/error.png + +logger: + level: DEBUG + +globals: + - id: init_in_progress + type: bool + restore_value: false + initial_value: "true" + - id: voice_assistant_phase + type: int + restore_value: false + initial_value: ${voice_assist_not_ready_phase_id} + - id: global_first_active_timer + type: voice_assistant::Timer + restore_value: false + - id: global_is_timer_active + type: bool + restore_value: false + - id: global_first_timer + type: voice_assistant::Timer + restore_value: false + - id: global_is_timer + type: bool + restore_value: false + +font: + - file: gfonts://Nanum+Gothic+Coding + id: clock_font + size: 160 + +color: + - id: black + hex: "000000" + + - id: white + hex: "FFFFFF" + +image: + - file: ${error_illustration_file} + id: kobold_error + type: RGB565 + - file: ${listening_illustration_file} + id: kobold_listening + type: RGB565 + - file: ${thinking_illustration_file} + id: kobold_thinking + type: RGB565 + - file: ${replying_illustration_file} + id: kobold_replying + type: RGB565 + - file: ${loading_illustration_file} + id: kobold_initializing + type: RGB565 + +external_components: + - source: + type: git + url: https://github.com/shelson/esphome-cst9217 + +########################## +# ____ # +# | _ \ # +# | |_) | __ _ ___ ___ # +# | _ < / _` / __|/ _ \ # +# | |_) | (_| \__ \ __/ # +# |____/ \__,_|___/\___| # +#Base##################### + +#time: +# - platform: homeassistant +# id: esptime +# on_time_sync: +# - script.execute: time_update +# on_time: +# - minutes: '*' +# seconds: 0 +# then: +# - script.execute: time_update + +api: + encryption: + key: + +ota: + - platform: esphome + password: + +wifi: + ssid: !secret wifi_ssid + password: !secret wifi_password + + ap: + ssid: + password: + +psram: + mode: octal + speed: 80MHz + +i2c: + sda: GPIO15 + scl: GPIO14 + scan: true + id: bus_a + +spi: + - id: spi_bus + clk_pin: GPIO2 + mosi_pin: GPIO1 + miso_pin: + number: GPIO3 + ignore_strapping_warning: true + - id: quad_spi_bus + type: quad + clk_pin: GPIO38 + data_pins: + - GPIO4 + - GPIO5 + - GPIO6 + - GPIO7 + +uart: + tx_pin: GPIO43 + rx_pin: GPIO44 + baud_rate: 256000 + rx_buffer_size: 1024 + parity: NONE + stop_bits: 1 + +ld2410: + +binary_sensor: + - platform: ld2410 + has_target: + name: Presence + has_moving_target: + name: Moving Target + has_still_target: + name: Still Target + out_pin_presence_status: + name: Out pin presence status + +sensor: + - platform: ld2410 + light: + name: Light + moving_distance: + name: Moving Distance + still_distance: + name: Still Distance + moving_energy: + name: Move Energy + still_energy: + name: Still Energy + detection_distance: + name: Detection Distance + g0: + move_energy: + name: G0 move energy + still_energy: + name: G0 still energy + g1: + move_energy: + name: G1 move energy + still_energy: + name: G1 still energy + g2: + move_energy: + name: G2 move energy + still_energy: + name: G2 still energy + g3: + move_energy: + name: G3 move energy + still_energy: + name: G3 still energy + g4: + move_energy: + name: G4 move energy + still_energy: + name: G4 still energy + g5: + move_energy: + name: G5 move energy + still_energy: + name: G5 still energy + g6: + move_energy: + name: G6 move energy + still_energy: + name: G6 still energy + g7: + move_energy: + name: G7 move energy + still_energy: + name: G7 still energy + g8: + move_energy: + name: G8 move energy + still_energy: + name: G8 still energy + + - platform: apds9960 + type: CLEAR + name: "APDS9960 Clear Channel" + +number: + - platform: ld2410 + timeout: + name: Timeout + light_threshold: + name: Light threshold + max_move_distance_gate: + name: Max move distance gate + max_still_distance_gate: + name: Max still distance gate + g0: + move_threshold: + name: G0 move threshold + still_threshold: + name: G0 still threshold + g1: + move_threshold: + name: G1 move threshold + still_threshold: + name: G1 still threshold + g2: + move_threshold: + name: G2 move threshold + still_threshold: + name: G2 still threshold + g3: + move_threshold: + name: G3 move threshold + still_threshold: + name: G3 still threshold + g4: + move_threshold: + name: G4 move threshold + still_threshold: + name: G4 still threshold + g5: + move_threshold: + name: G5 move threshold + still_threshold: + name: G5 still threshold + g6: + move_threshold: + name: G6 move threshold + still_threshold: + name: G6 still threshold + g7: + move_threshold: + name: G7 move threshold + still_threshold: + name: G7 still threshold + g8: + move_threshold: + name: G8 move threshold + still_threshold: + name: G8 still threshold + +apds9960: + address: 0x39 + update_interval: 10s + ambient_light_gain: 64x + gesture_gain: 1x + gesture_led_drive: 100ma + gesture_wait_time: 39.2ms + +select: + - platform: template + entity_category: config + name: Wake word engine location + id: wake_word_engine_location + icon: "mdi:account-voice" + optimistic: true + restore_value: true + options: + - In Home Assistant + - On device + initial_option: On device + on_value: + - if: + condition: + lambda: return !id(init_in_progress); + then: + - wait_until: + lambda: return id(voice_assistant_phase) == ${voice_assist_muted_phase_id} || id(voice_assistant_phase) == ${voice_assist_idle_phase_id}; + - if: + condition: + lambda: return x == "In Home Assistant"; + then: + - micro_wake_word.stop + - delay: 500ms + - if: + condition: + switch.is_off: mute + then: + - lambda: id(va).set_use_wake_word(true); + - voice_assistant.start_continuous: + - if: + condition: + lambda: return x == "On device"; + then: + - lambda: id(va).set_use_wake_word(false); + - voice_assistant.stop + - delay: 500ms + - if: + condition: + switch.is_off: mute + then: + - micro_wake_word.start + + +################################# +# _ _ # +# /\ | (_) # +# / \ _ _ __| |_ ___ # +# / /\ \| | | |/ _` | |/ _ \ # +# / ____ \ |_| | (_| | | (_) | # +# /_/ \_\__,_|\__,_|_|\___/ # +#Audio########################### + +i2s_audio: + - id: i2s_audio_bus + i2s_mclk_pin: GPIO42 + i2s_bclk_pin: GPIO9 + i2s_lrclk_pin: + number: GPIO45 + ignore_strapping_warning: true + +audio_adc: + - platform: es7210 + id: es7210_adc + bits_per_sample: $i2s_bps_mic + sample_rate: $i2s_sample_rate_mic + +audio_dac: + - platform: es8311 + id: es8311_dac + bits_per_sample: $i2s_bps_spk + sample_rate: $i2s_sample_rate_spk + +microphone: + - platform: i2s_audio + id: box_mic + sample_rate: $i2s_sample_rate_mic + i2s_din_pin: GPIO10 + bits_per_sample: $i2s_bps_mic + adc_type: external + +speaker: + - platform: i2s_audio + id: box_speaker + i2s_dout_pin: GPIO8 + dac_type: external + sample_rate: $i2s_sample_rate_spk + bits_per_sample: $i2s_bps_spk + audio_dac: es8311_dac + buffer_duration: 90ms + use_apll: true + +media_player: + - platform: speaker + name: player + id: speaker_media_player + volume_max: 80% + announcement_pipeline: + speaker: box_speaker + format: FLAC + on_announcement: + - if: + condition: + - microphone.is_capturing: + then: + - script.execute: stop_wake_word + # Ensure VA stops before moving on + - if: + condition: + - lambda: return id(wake_word_engine_location).state == "In Home Assistant"; + then: + - wait_until: + - not: + voice_assistant.is_running: + - if: + condition: + not: + voice_assistant.is_running: + then: + - lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id}; + - logger.log: "WEH WEH!" + on_idle: + # Since VA isn't running, this is the end of user-intiated media playback. Restart the wake word. + - if: + condition: + - and: + - not: + voice_assistant.is_running: + - lambda: return id(voice_assistant_phase) != ${voice_assist_idle_phase_id}; + then: + - script.execute: start_wake_word + - script.execute: set_idle_or_mute_phase + +###################################### +# _____ _ _ # +# | __ \(_) | | # +# | | | |_ ___ _ __ | | __ _ _ _ # +# | | | | / __| '_ \| |/ _` | | | | # +# | |__| | \__ \ |_) | | (_| | |_| | # +# |_____/|_|___/ .__/|_|\__,_|\__, | # +# | | __/ | # +# |_| |___/ # +#Display############################## + +lvgl: + id: voice_assistant_display + buffer_size: 25% + pages: + - id: image_page + bg_color: black + widgets: + - image: + id: current_image + align: CENTER + src: kobold_initializing + - id: clock_page + bg_color: black + +display: + - platform: mipi_spi + id: disp1 + model: CO5300 + bus_mode: quad + reset_pin: GPIO39 + cs_pin: GPIO12 + data_rate: 80MHz + dimensions: + height: 466 + width: 466 + offset_width: 6 + +light: + - platform: monochromatic + id: display_backlight + name: "Backlight" + output: backlight_brightness + default_transition_length: + milliseconds: 0 + initial_state: + brightness: 81% + restore_mode: + ALWAYS_ON + +output: + - platform: template + id: backlight_brightness + type: float + write_action: + then: + - lambda: |- + id(disp1).set_brightness(state*255); + +touchscreen: + - platform: cst9217 + display: disp1 + id: ts_disp1 + interrupt_pin: GPIO11 + reset_pin: GPIO40 + transform: + mirror_x: true + mirror_y: true + on_update: + - lambda: |- + for (auto touch: touches) { + if (touch.state <= 2) { + ESP_LOGI("Touch points:", "id=%d x=%d, y=%d", touch.id, touch.x, touch.y); + } + } + +############################################ +# _____ _ _ # +# |_ _| | | | | # +# | | _ __ | |_ ___ _ __ __ _ ___| |_ # +# | | | '_ \| __/ _ \ '__/ _` |/ __| __| # +# _| |_| | | | || __/ | | (_| | (__| |_ # +# |_____|_| |_|\__\___|_| \__,_|\___|\__| # +#Interact################################### + +button: + - platform: restart + name: "Living Room Restart" + +switch: + - platform: ld2410 + engineering_mode: + name: Engineering mode + bluetooth: + name: Control bluetooth + + - platform: gpio + name: "Speaker Enable" + pin: + number: GPIO46 + ignore_strapping_warning: true + restore_mode: RESTORE_DEFAULT_ON + + - platform: template + name: Mute + id: mute + icon: "mdi:microphone-off" + optimistic: true + restore_mode: RESTORE_DEFAULT_OFF + entity_category: config + on_turn_off: + - microphone.unmute: + - lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id}; + - logger.log: "Calling draw_display from switch on_turn_off" + on_turn_on: + - microphone.mute: + - lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id}; + - logger.log: "Calling draw_display from switch on_turn_on" + + +micro_wake_word: + id: mww + models: + - model: https://raw.githubusercontent.com/YOUR-WORST-TACO/CustomWakeupWords/refs/heads/main/models/hey_nokari/hey_nokari.json + id: hey_nokari + # probability_cutoff: 0.98 + # sliding_window_size: 5 + - model: https://raw.githubusercontent.com/YOUR-WORST-TACO/CustomWakeupWords/refs/heads/main/models/nokari/nokari.json + id: nokari + probability_cutoff: 0.98 + sliding_window_size: 5 + on_wake_word_detected: + - voice_assistant.start: + wake_word: !lambda return wake_word; + +voice_assistant: + id: va + microphone: box_mic + media_player: speaker_media_player + micro_wake_word: mww + noise_suppression_level: 2 + auto_gain: 31dBFS + volume_multiplier: 2.0 + on_listening: + - if: + condition: + lambda: return id(voice_assistant_phase) != ${voice_assist_listening_phase_id}; + then: + - lambda: id(voice_assistant_phase) = ${voice_assist_listening_phase_id}; + - logger.log: "Calling draw_display from voice_assistant on_listening" + - script.execute: draw_display + - text_sensor.template.publish: + id: text_request + state: "..." + - text_sensor.template.publish: + id: text_response + state: "..." + on_stt_vad_end: + - lambda: id(voice_assistant_phase) = ${voice_assist_thinking_phase_id}; + - logger.log: "Calling draw_display from voice_assistant on_stt_vad_end" + - script.execute: draw_display + on_stt_end: + - text_sensor.template.publish: + id: text_request + state: !lambda return x; + - logger.log: "Calling draw_display from voice_assistant on_stt_end" + - script.execute: draw_display + on_tts_start: + - text_sensor.template.publish: + id: text_response + state: !lambda return x; + - lambda: id(voice_assistant_phase) = ${voice_assist_replying_phase_id}; + - logger.log: "Calling draw_display from voice_assistant on_tts_start" + - script.execute: draw_display + on_end: + # Wait a short amount of time to see if an announcement starts + - wait_until: + condition: + - media_player.is_announcing: + timeout: 0.5s + # Announcement is finished and the I2S bus is free + - wait_until: + - and: + - not: + media_player.is_announcing: + - not: + speaker.is_playing: + # Restart only mWW if enabled; streaming wake words automatically restart + - if: + condition: + - lambda: return id(wake_word_engine_location).state == "On device"; + then: + - lambda: id(va).set_use_wake_word(false); + - micro_wake_word.start: + - script.execute: set_idle_or_mute_phase + - logger.log: "Calling draw_display from voice_assistant on_end" + - script.execute: draw_display + # Clear text sensors + - text_sensor.template.publish: + id: text_request + state: "" + - text_sensor.template.publish: + id: text_response + state: "" + on_error: + - if: + condition: + lambda: return !id(init_in_progress); + then: + - lambda: id(voice_assistant_phase) = ${voice_assist_error_phase_id}; + - logger.log: "Calling draw_display from voice_assistant on_error" + - script.execute: draw_display + - delay: 1s + - if: + condition: + switch.is_off: mute + then: + - lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id}; + else: + - lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id}; + - logger.log: "Calling draw_display from voice_assistant on_error" + - script.execute: draw_display + on_client_connected: + - lambda: id(init_in_progress) = false; + - script.execute: start_wake_word + - script.execute: set_idle_or_mute_phase + - logger.log: "Calling draw_display from voice_assistant on_client_connected" + - script.execute: draw_display + on_client_disconnected: + - script.execute: stop_wake_word + - lambda: id(voice_assistant_phase) = ${voice_assist_not_ready_phase_id}; + - logger.log: "Calling draw_display from voice_assistant on_client_disconnected" + - script.execute: draw_display + +text_sensor: + - id: text_request + platform: template + on_value: + lambda: |- + if(id(text_request).state.length()>32) { + std::string name = id(text_request).state.c_str(); + std::string truncated = esphome::str_truncate(name.c_str(),31); + id(text_request).state = (truncated+"...").c_str(); + } + - id: text_response + platform: template + on_value: + lambda: |- + if(id(text_response).state.length()>32) { + std::string name = id(text_response).state.c_str(); + std::string truncated = esphome::str_truncate(name.c_str(),31); + id(text_response).state = (truncated+"...").c_str(); + } + +mapping: + - id: image_mapping + from: int + to: image + entries: + ${voice_assist_listening_phase_id}: kobold_listening + ${voice_assist_thinking_phase_id}: kobold_thinking + ${voice_assist_replying_phase_id}: kobold_replying + ${voice_assist_not_ready_phase_id}: kobold_error + ${voice_assist_error_phase_id}: kobold_error + ${voice_assist_muted_phase_id}: kobold_error + +script: +# - id: time_update +# then: +# - lvgl.animation.start: text_fade_out +# - delay: 1s +# - lvgl.label.update: +# id: lbl_time +# text: !lambda |- +# return id(esptime).now().strftime("%H:%M"); +# - lvgl.animation.start: text_fade_in +# - logger.log: "update Time script ran" + + - id: draw_display + then: + - logger.log: "Drawing display" + - if: + condition: + lambda: return !id(init_in_progress); + then: + - if: + condition: + wifi.connected: + then: + - if: + condition: + api.connected: + then: + - if: + condition: + lambda: return id(voice_assistant_phase) == ${voice_assist_idle_phase_id}; + then: + - lvgl.page.show: + id: clock_page + animation: OUT_BOTTOM + time: 500ms + else: + - lvgl.page.show: + id: image_page + animation: OUT_TOP + time: 500ms + - lvgl.image.update: + id: current_image + src: !lambda return id(image_mapping)[id(voice_assistant_phase)]; + else: + - lvgl.page.show: + id: image_page + animation: OUT_TOP + time: 500ms + - lvgl.image.update: + id: current_image + src: kobold_error + else: + - lvgl.page.show: + id: image_page + animation: OUT_TOP + time: 500ms + - lvgl.image.update: + id: current_image + src: kobold_error + else: + - lvgl.page.show: + id: image_page + animation: OUT_TOP + time: 500ms + - lvgl.image.update: + id: current_image + src: kobold_initializing + + # Starts either mWW or the streaming wake word, depending on the configured location + - id: start_wake_word + then: + - if: + condition: + and: + - not: + - voice_assistant.is_running: + - lambda: return id(wake_word_engine_location).state == "On device"; + then: + - lambda: id(va).set_use_wake_word(false); + - micro_wake_word.start: + - if: + condition: + and: + - not: + - voice_assistant.is_running: + - lambda: return id(wake_word_engine_location).state == "In Home Assistant"; + then: + - lambda: id(va).set_use_wake_word(true); + - voice_assistant.start_continuous: + # Stops either mWW or the streaming wake word, depending on the configured location + - id: stop_wake_word + then: + - if: + condition: + lambda: return id(wake_word_engine_location).state == "In Home Assistant"; + then: + - lambda: id(va).set_use_wake_word(false); + - voice_assistant.stop: + - if: + condition: + lambda: return id(wake_word_engine_location).state == "On device"; + then: + - micro_wake_word.stop: + # Set the voice assistant phase to idle or muted, depending on if the software mute switch is activated + - id: set_idle_or_mute_phase + then: + - if: + condition: + switch.is_off: mute + then: + - lambda: id(voice_assistant_phase) = ${voice_assist_idle_phase_id}; + else: + - lambda: id(voice_assistant_phase) = ${voice_assist_muted_phase_id};