Files
led-controller/src/util/audio_detector.py
Jimmy ace5770b3a refactor(api): complete fastapi migration and related features
Finish native FastAPI controllers, drop vendored microdot, and add
Wi-Fi driver runtime, beat SSE, simulated BPM, sequence playback
improvements, bridge ESP-NOW sources, UI updates, and tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-06-11 22:55:28 +12:00

735 lines
26 KiB
Python

import collections
import importlib.util
import os
import queue
import threading
import time
from typing import Any
_HOLDOVER_MAX_S = 300.0
# After this many seconds without a detected beat, re-prime aubio and start BPM holdover
# (same window as status() uses to hide stale BPM).
_SILENCE_GAP_S = 4.0
class AudioBeatDetector:
def __init__(self):
self._lock = threading.Lock()
self._thread = None
self._stream = None
self._running = False
self._stop_event = threading.Event()
self._runtime = None
self._pending_reset = False
self._holdover_thread: threading.Thread | None = None
self._holdover_stop = threading.Event()
self._holdover_active = False
self._last_real_beat_ts: float | None = None
self._last_gap_tempo_reset_ts: float = 0.0
self._status = {
"running": False,
"bpm": None,
"last_beat_ts": None,
"beat_seq": 0,
"beat_type": "unknown",
"beat_type_confidence": 0.0,
"bar_beat": 1,
"beats_per_bar": 4,
"is_downbeat": False,
"phase_confidence": 0.0,
"bar_phase_readout": "1/4",
"error": None,
"device": None,
"input_level": 0.0,
}
def list_input_devices(self):
try:
from util.pulse_audio_devices import list_pulse_matched_input_devices
pulse = list_pulse_matched_input_devices()
if pulse:
return pulse
except Exception as e:
print(f"[audio] pulse device list skipped: {e!r}")
sd_list = self._list_sounddevice_input_devices()
if sd_list:
print("[audio] device list: sounddevice fallback (install/use pactl for Pulse names)")
return sd_list
@staticmethod
def _skip_sounddevice_virtual(name: str, hostapi_name: str) -> bool:
"""Hide PortAudio/Pulse aggregate devices (pipewire, pulse, default)."""
n = name.strip().lower()
if n in ("pipewire", "pulse", "default", "sysdefault"):
return True
ha = hostapi_name.strip().lower()
if ha in ("pulse", "pipewire") and n in ("default", "pipewire", "pulse"):
return True
return False
def _list_sounddevice_input_devices(self):
import sounddevice as sd
devices = sd.query_devices()
hostapis = sd.query_hostapis()
default_input_idx = None
try:
default_input_idx = int(sd.default.device[0])
except Exception:
default_input_idx = None
out = []
for idx, dev in enumerate(devices):
name = str(dev.get("name", f"Input {idx}"))
chans = int(dev.get("max_input_channels", 0))
is_monitor_named = "monitor" in name.lower()
hostapi_idx = int(dev.get("hostapi", -1))
hostapi_name = (
str(hostapis[hostapi_idx].get("name", "unknown"))
if 0 <= hostapi_idx < len(hostapis)
else "unknown"
)
if self._skip_sounddevice_virtual(name, hostapi_name):
continue
if chans <= 0 and not is_monitor_named:
continue
sr = int(dev.get("default_samplerate", 44100))
is_default = default_input_idx is not None and idx == default_input_idx
ch_label = f"{chans}ch" if chans > 0 else "0ch?"
label = f"[{idx}] {name} ({ch_label} @ {sr}Hz, {hostapi_name})"
if is_default:
label = f"{label} [default]"
if is_monitor_named:
label = f"{label} [monitor]"
display_name = name
if is_default:
display_name = f"{display_name} (default)"
out.append(
{
"id": idx,
"name": name,
"display_name": display_name,
"label": label,
"max_input_channels": chans,
"default_samplerate": sr,
"is_default": is_default,
"hostapi": hostapi_name,
}
)
return out
def diagnostics(self):
import sounddevice as sd
devices = sd.query_devices()
hostapis = sd.query_hostapis()
default_input = None
try:
default_input = sd.default.device[0]
except Exception:
default_input = None
return {
"default_input": default_input,
"hostapis": hostapis,
"devices": devices,
}
def start(self, device=None):
try:
from util.pulse_audio_devices import resolve_capture_device
device = resolve_capture_device(device)
except Exception as e:
self._set_error(str(e))
raise
should_restart = False
with self._lock:
should_restart = self._running
if should_restart:
self.stop()
with self._lock:
self._stop_event.clear()
self._last_real_beat_ts = None
self._last_gap_tempo_reset_ts = 0.0
self._status.update(
{
"running": True,
"bpm": None,
"last_beat_ts": None,
"beat_seq": 0,
"beat_type": "unknown",
"beat_type_confidence": 0.0,
"bar_beat": 1,
"beats_per_bar": 4,
"is_downbeat": False,
"phase_confidence": 0.0,
"bar_phase_readout": "1/4",
"error": None,
"device": device,
}
)
self._running = True
self._thread = threading.Thread(
target=self._run_loop, args=(device,), daemon=True
)
self._thread.start()
def stop(self):
self._stop_bpm_holdover()
with self._lock:
self._stop_event.set()
t = self._thread
stream = self._stream
try:
import sounddevice as sd
sd.stop(ignore_errors=True)
except Exception:
pass
if stream is not None:
try:
stream.abort()
except Exception:
pass
try:
stream.stop()
except Exception:
pass
try:
stream.close()
except Exception:
pass
if t and t.is_alive():
t.join(timeout=3.0)
with self._lock:
self._running = False
self._thread = None
self._stream = None
self._pending_reset = False
self._last_real_beat_ts = None
self._last_gap_tempo_reset_ts = 0.0
self._status["running"] = False
self._status["input_level"] = 0.0
def _update_input_level(self, mono) -> None:
import numpy as np
arr = np.asarray(mono, dtype=np.float32)
if arr.size == 0:
inst = 0.0
else:
peak = float(np.max(np.abs(arr)))
rms = float(np.sqrt(np.mean(arr * arr)))
inst = min(1.0, max(peak, rms * 2.0))
with self._lock:
prev = float(self._status.get("input_level") or 0.0)
if inst >= prev:
self._status["input_level"] = inst
else:
self._status["input_level"] = max(inst, prev * 0.82)
def _decay_input_level(self) -> None:
with self._lock:
prev = float(self._status.get("input_level") or 0.0)
self._status["input_level"] = prev * 0.82
def _input_gain(self) -> float:
try:
from settings import get_settings
vol = int(get_settings().get("audio_input_volume") or 100)
except (TypeError, ValueError, ImportError):
vol = 100
vol = max(0, min(200, vol))
return vol / 100.0
def status(self):
with self._lock:
st = dict(self._status)
holdover = self._holdover_active
last = st.get("last_beat_ts")
if st.get("running") and last is not None and not holdover:
try:
if (time.time() - float(last)) > 4.0:
st["bpm"] = None
except (TypeError, ValueError):
pass
if st.get("bpm") is not None:
from util.bpm_limits import clamp_bpm_optional
st["bpm"] = clamp_bpm_optional(st["bpm"])
return st
def _apply_tracking_reset_status(self) -> None:
"""Refresh published status after a tracking reset (lock must be held)."""
bpb = max(1, int(self._status.get("beats_per_bar") or 4))
self._status.update(
{
"running": True,
"beat_type": "unknown",
"beat_type_confidence": 0.0,
"bar_beat": 1,
"is_downbeat": True,
"phase_confidence": 0.0,
"bar_phase_readout": f"1/{bpb}",
}
)
def _clamp_holdover_bpm(self, bpm: Any) -> float | None:
from util.bpm_limits import clamp_bpm_optional
return clamp_bpm_optional(bpm)
def _holdover_interval_s(self, bpm: float) -> float:
from util.bpm_limits import clamp_bpm
return 60.0 / clamp_bpm(bpm)
def _stop_bpm_holdover(self) -> None:
with self._lock:
self._holdover_active = False
self._holdover_stop.set()
t = self._holdover_thread
if t and t.is_alive() and t is not threading.current_thread():
t.join(timeout=2.0)
with self._lock:
if self._holdover_thread is t:
self._holdover_thread = None
def _advance_holdover_bar_phase_locked(self) -> dict:
"""Advance bar phase for one synthetic beat (lock must be held)."""
bpb = max(1, int(self._status.get("beats_per_bar") or 4))
prev = int(self._status.get("bar_beat") or 1)
bar_beat = (prev % bpb) + 1
is_downbeat = bar_beat == 1
bar_readout = f"{bar_beat}/{bpb}"
self._status["bar_beat"] = bar_beat
self._status["is_downbeat"] = is_downbeat
self._status["bar_phase_readout"] = bar_readout
return {
"bar_beat": bar_beat,
"beats_per_bar": bpb,
"is_downbeat": is_downbeat,
"bar_phase_readout": bar_readout,
}
def _emit_holdover_beat(self, bpm: float) -> None:
now = time.time()
with self._lock:
if not self._running or not self._holdover_active:
return
self._advance_holdover_bar_phase_locked()
self._status["last_beat_ts"] = now
self._status["bpm"] = float(bpm)
self._status["beat_type"] = "holdover"
self._status["beat_type_confidence"] = 0.0
self._status["beat_seq"] = int(self._status.get("beat_seq", 0)) + 1
try:
from util import sequence_playback as seq_pb
seq_pb.push_thread_beat()
except Exception as e:
print(f"[audio] holdover beat queue: {e}")
def _holdover_loop(self, bpm: float, started_at: float) -> None:
interval = self._holdover_interval_s(bpm)
while not self._holdover_stop.is_set():
with self._lock:
if not self._running or not self._holdover_active:
return
if (time.time() - started_at) > _HOLDOVER_MAX_S:
self._holdover_active = False
return
last = self._status.get("last_beat_ts")
if last is not None:
try:
delay = max(0.02, float(last) + interval - time.time())
except (TypeError, ValueError):
delay = interval
else:
delay = interval
if self._holdover_stop.wait(delay):
return
self._emit_holdover_beat(bpm)
def prime_bpm_holdover(self, bpm: float) -> None:
"""Public: tick at *bpm* until the next detected beat (e.g. pending sequence switch)."""
if not self._running:
return
self._start_bpm_holdover(bpm)
def _start_bpm_holdover(self, bpm: float) -> None:
bpm_v = self._clamp_holdover_bpm(bpm)
if bpm_v is None:
return
self._stop_bpm_holdover()
self._holdover_stop.clear()
started_at = time.time()
with self._lock:
self._holdover_active = True
self._holdover_thread = threading.Thread(
target=self._holdover_loop,
args=(bpm_v, started_at),
name="audio-bpm-holdover",
daemon=True,
)
t = self._holdover_thread
t.start()
def _process_pending_reset(self, runtime) -> None:
"""Run ``reset_state`` on the audio thread (safe for aubio tempo)."""
with self._lock:
if not self._pending_reset:
return
self._pending_reset = False
try:
runtime.reset_state()
with self._lock:
self._apply_tracking_reset_status()
except Exception as e:
print(f"[audio] pending reset: {e}")
def reset_tracking(self) -> bool:
"""Clear detector tempo history without stopping the input stream."""
holdover_bpm = None
with self._lock:
if not self._running or self._runtime is None:
return False
holdover_bpm = self._clamp_holdover_bpm(self._status.get("bpm"))
self._pending_reset = True
self._apply_tracking_reset_status()
if holdover_bpm is not None:
self._start_bpm_holdover(holdover_bpm)
return True
def _set_error(self, msg):
print(f"[audio] {msg}")
with self._lock:
self._status["error"] = msg
self._status["running"] = False
self._running = False
def anchor_bar_phase(self) -> bool:
"""Mark the current moment as bar beat 1 (downbeat), e.g. after manual sync."""
with self._lock:
rt = self._runtime
if rt is None:
return False
try:
rt.anchor_bar_phase(time.time())
with self._lock:
self._status["bar_beat"] = 1
self._status["is_downbeat"] = True
self._status["bar_phase_readout"] = f"1/{int(self._status.get('beats_per_bar') or 4)}"
self._status["phase_confidence"] = max(
float(self._status.get("phase_confidence") or 0.0), 0.85
)
return True
except Exception as e:
print(f"[audio] anchor_bar_phase: {e}")
return False
def _maybe_recover_after_silence_gap(self, runtime) -> None:
"""After a quiet spell, reset tempo tracking and run holdover until real beats return."""
now = time.time()
with self._lock:
if not self._running:
return
last_real = self._last_real_beat_ts
bpm = self._clamp_holdover_bpm(self._status.get("bpm"))
holdover = self._holdover_active
last_reset = self._last_gap_tempo_reset_ts
if last_real is None:
return
if bpm is None:
from util.bpm_limits import clamp_bpm
bpm = clamp_bpm(120)
try:
gap = now - float(last_real)
except (TypeError, ValueError):
return
if gap < _SILENCE_GAP_S:
return
if not holdover:
self._start_bpm_holdover(bpm)
try:
since_reset = (
now - float(last_reset) if last_reset else _SILENCE_GAP_S
)
except (TypeError, ValueError):
since_reset = _SILENCE_GAP_S
if since_reset >= _SILENCE_GAP_S:
try:
runtime.reset_tempo_state()
except Exception as e:
print(f"[audio] silence gap tempo reset: {e}")
else:
with self._lock:
self._last_gap_tempo_reset_ts = now
def _record_beat(self, bpm, beat_type="unknown", beat_type_confidence=0.0, **phase_fields):
from util.bpm_limits import clamp_bpm_optional
self._stop_bpm_holdover()
now = time.time()
self._last_real_beat_ts = now
bpm = clamp_bpm_optional(bpm)
with self._lock:
if bpm is None:
bpm = clamp_bpm_optional(self._status.get("bpm"))
self._last_gap_tempo_reset_ts = 0.0
self._status["last_beat_ts"] = now
self._status["bpm"] = bpm
self._status["beat_type"] = beat_type
self._status["beat_type_confidence"] = float(beat_type_confidence or 0.0)
self._status["beat_seq"] = int(self._status.get("beat_seq", 0)) + 1
if phase_fields.get("bar_beat") is not None:
self._status["bar_beat"] = int(phase_fields["bar_beat"])
if phase_fields.get("beats_per_bar") is not None:
self._status["beats_per_bar"] = int(phase_fields["beats_per_bar"])
if phase_fields.get("is_downbeat") is not None:
self._status["is_downbeat"] = bool(phase_fields["is_downbeat"])
if phase_fields.get("phase_confidence") is not None:
self._status["phase_confidence"] = float(phase_fields["phase_confidence"])
if phase_fields.get("bar_phase_readout"):
self._status["bar_phase_readout"] = str(phase_fields["bar_phase_readout"])
try:
from util import sequence_playback as seq_pb
seq_pb.push_thread_beat()
except Exception as e:
print(f"[audio] sequence beat queue: {e}")
holdover_bpm = None
with self._lock:
if self._running:
holdover_bpm = self._clamp_holdover_bpm(self._status.get("bpm"))
if holdover_bpm is not None:
self._start_bpm_holdover(holdover_bpm)
def _run_loop(self, device):
try:
import argparse
import numpy as np
import sounddevice as sd
except Exception as e:
self._set_error(f"audio deps unavailable: {e}")
return
try:
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
beat_detect_path = os.path.join(root_dir, "tests", "beat_detect.py")
spec = importlib.util.spec_from_file_location("beat_detect_runtime", beat_detect_path)
if spec is None or spec.loader is None:
raise RuntimeError("cannot load tests/beat_detect.py")
beat_mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(beat_mod)
from util.pulse_audio_devices import resolve_capture_device
device = resolve_capture_device(device)
if device is None:
try:
device = int(sd.default.device[0])
except Exception:
device = -1
if device is None or device < 0:
raise RuntimeError(
"no default input device; open Audio, pick an input, then Start"
)
if not isinstance(device, int):
raise RuntimeError(
f"internal error: unresolved capture device {device!r}"
)
dev_info = sd.query_devices(device, "input")
sample_rate = int(dev_info["default_samplerate"])
from util.bpm_limits import max_beat_min_ioi_ms
args = argparse.Namespace(
mode="aubio",
device=device,
sample_rate=sample_rate,
hop_size=256,
win_mult=2,
min_band_hz=45.0,
max_band_hz=180.0,
energy_weight=0.7,
flux_weight=0.3,
threshold_multiplier=1.35,
ema_alpha=0.08,
min_ioi_ms=max_beat_min_ioi_ms(),
bpm_window=8,
post_url="",
aubio_method="default",
aubio_threshold=0.14,
beats_per_bar=4,
)
runtime = beat_mod.BeatDetectRuntime(args)
runtime.setup(sample_rate=sample_rate)
with self._lock:
self._runtime = runtime
hop_size = runtime.frame_size
audio_q = queue.Queue(maxsize=64)
def callback(indata, frames, _time_info, status):
_ = frames
if status:
print(f"[audio] status: {status}")
mono = np.asarray(indata[:, 0], dtype=np.float32)
if not audio_q.full():
audio_q.put_nowait(mono)
stream = sd.InputStream(
device=device,
channels=1,
samplerate=sample_rate,
blocksize=hop_size,
callback=callback,
)
with self._lock:
self._stream = stream
stream.start()
try:
while not self._stop_event.is_set():
self._process_pending_reset(runtime)
try:
frame = audio_q.get(timeout=0.1)
except queue.Empty:
self._decay_input_level()
self._maybe_recover_after_silence_gap(runtime)
continue
self._process_pending_reset(runtime)
if frame.shape[0] != hop_size:
if frame.shape[0] > hop_size:
frame = frame[:hop_size]
else:
frame = np.pad(frame, (0, hop_size - frame.shape[0]))
gain = self._input_gain()
if gain != 1.0:
frame = frame * gain
self._update_input_level(frame)
event = runtime.process_frame(frame, now_s=time.time())
if event is None:
self._maybe_recover_after_silence_gap(runtime)
continue
bpm = event.get("bpm")
self._record_beat(
bpm,
beat_type=event.get("beat_type", "unknown"),
beat_type_confidence=event.get("beat_type_confidence", 0.0),
bar_beat=event.get("bar_beat"),
beats_per_bar=event.get("beats_per_bar"),
is_downbeat=event.get("is_downbeat"),
phase_confidence=event.get("phase_confidence"),
bar_phase_readout=event.get("bar_phase_readout"),
)
finally:
try:
stream.stop()
except Exception:
pass
try:
stream.close()
except Exception:
pass
with self._lock:
if self._stream is stream:
self._stream = None
except Exception as e:
self._set_error(f"detector failed: {e}")
return
finally:
with self._lock:
self._running = False
self._status["running"] = False
self._runtime = None
# Set from ``main`` so sequence playback can tell real audio from simulated beats.
_shared_beat_detector = None
def set_shared_beat_detector(det):
global _shared_beat_detector
_shared_beat_detector = det
def shared_beat_detector_running():
d = _shared_beat_detector
if d is None:
return False
try:
return bool(d.status().get("running"))
except Exception:
return False
def shared_beat_detector_timing_sequences() -> bool:
"""True when live audio is running and has clocked a beat recently enough to drive sequences."""
d = _shared_beat_detector
if d is None:
return False
try:
st = dict(d.status())
except Exception:
return False
if not st.get("running"):
return False
with d._lock:
last = d._last_real_beat_ts
holdover = d._holdover_active
if holdover:
return True
if last is None:
return False
try:
gap = time.time() - float(last)
except (TypeError, ValueError):
return False
from util.bpm_limits import clamp_bpm
bpm_raw = st.get("bpm")
try:
bpm_v = clamp_bpm(bpm_raw) if bpm_raw is not None else 120.0
except (TypeError, ValueError):
bpm_v = 120.0
max_gap = (60.0 / bpm_v) * 2.0
return gap < max_gap
def shared_beat_status_snapshot() -> dict:
"""Thread-safe copy of live detector status, or {} if audio is off."""
d = _shared_beat_detector
if d is None:
return {}
try:
return dict(d.status())
except Exception:
return {}
def prime_bpm_holdover(bpm: float) -> None:
"""Start BPM holdover on the shared detector when audio is on but not clocking."""
d = _shared_beat_detector
if d is None:
return
try:
d.prime_bpm_holdover(bpm)
except Exception:
pass
def anchor_shared_bar_phase() -> bool:
"""Anchor bar phase on the shared detector (no-op if audio is off)."""
d = _shared_beat_detector
if d is None:
return False
try:
return bool(d.anchor_bar_phase())
except Exception:
return False