feat(audio): move beat routing server-side and extend presets
Route beat-triggered manual selects from the controller server, add preset background and beat-counter UI support, and bump led-driver to include the matching pattern/runtime fixes. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
282
src/util/audio_detector.py
Normal file
282
src/util/audio_detector.py
Normal file
@@ -0,0 +1,282 @@
|
||||
import collections
|
||||
import importlib.util
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
class AudioBeatDetector:
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._thread = None
|
||||
self._stream = None
|
||||
self._running = False
|
||||
self._stop_event = threading.Event()
|
||||
self._status = {
|
||||
"running": False,
|
||||
"bpm": None,
|
||||
"last_beat_ts": None,
|
||||
"beat_seq": 0,
|
||||
"beat_type": "unknown",
|
||||
"beat_type_confidence": 0.0,
|
||||
"error": None,
|
||||
"device": None,
|
||||
}
|
||||
|
||||
def list_input_devices(self):
|
||||
import sounddevice as sd
|
||||
|
||||
devices = sd.query_devices()
|
||||
hostapis = sd.query_hostapis()
|
||||
default_input_idx = None
|
||||
try:
|
||||
default_input_idx = int(sd.default.device[0])
|
||||
except Exception:
|
||||
default_input_idx = None
|
||||
out = []
|
||||
for idx, dev in enumerate(devices):
|
||||
name = str(dev.get("name", f"Input {idx}"))
|
||||
chans = int(dev.get("max_input_channels", 0))
|
||||
is_monitor_named = "monitor" in name.lower()
|
||||
if chans <= 0 and not is_monitor_named:
|
||||
continue
|
||||
sr = int(dev.get("default_samplerate", 44100))
|
||||
hostapi_idx = int(dev.get("hostapi", -1))
|
||||
hostapi_name = (
|
||||
str(hostapis[hostapi_idx].get("name", "unknown"))
|
||||
if 0 <= hostapi_idx < len(hostapis)
|
||||
else "unknown"
|
||||
)
|
||||
is_default = default_input_idx is not None and idx == default_input_idx
|
||||
ch_label = f"{chans}ch" if chans > 0 else "0ch?"
|
||||
label = f"[{idx}] {name} ({ch_label} @ {sr}Hz, {hostapi_name})"
|
||||
if is_default:
|
||||
label = f"{label} [default]"
|
||||
if is_monitor_named:
|
||||
label = f"{label} [monitor]"
|
||||
out.append(
|
||||
{
|
||||
"id": idx,
|
||||
"name": name,
|
||||
"label": label,
|
||||
"max_input_channels": chans,
|
||||
"default_samplerate": sr,
|
||||
"is_default": is_default,
|
||||
"hostapi": hostapi_name,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
def diagnostics(self):
|
||||
import sounddevice as sd
|
||||
|
||||
devices = sd.query_devices()
|
||||
hostapis = sd.query_hostapis()
|
||||
default_input = None
|
||||
try:
|
||||
default_input = sd.default.device[0]
|
||||
except Exception:
|
||||
default_input = None
|
||||
return {
|
||||
"default_input": default_input,
|
||||
"hostapis": hostapis,
|
||||
"devices": devices,
|
||||
}
|
||||
|
||||
def start(self, device=None):
|
||||
should_restart = False
|
||||
with self._lock:
|
||||
should_restart = self._running
|
||||
if should_restart:
|
||||
self.stop()
|
||||
with self._lock:
|
||||
self._stop_event.clear()
|
||||
self._status.update(
|
||||
{
|
||||
"running": True,
|
||||
"bpm": None,
|
||||
"last_beat_ts": None,
|
||||
"beat_seq": 0,
|
||||
"beat_type": "unknown",
|
||||
"beat_type_confidence": 0.0,
|
||||
"error": None,
|
||||
"device": device,
|
||||
}
|
||||
)
|
||||
self._running = True
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_loop, args=(device,), daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self):
|
||||
with self._lock:
|
||||
self._stop_event.set()
|
||||
t = self._thread
|
||||
stream = self._stream
|
||||
try:
|
||||
import sounddevice as sd
|
||||
sd.stop(ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
if stream is not None:
|
||||
try:
|
||||
stream.abort()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
stream.stop()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
if t and t.is_alive():
|
||||
t.join(timeout=3.0)
|
||||
with self._lock:
|
||||
self._running = False
|
||||
self._thread = None
|
||||
self._stream = None
|
||||
self._status["running"] = False
|
||||
|
||||
def status(self):
|
||||
with self._lock:
|
||||
return dict(self._status)
|
||||
|
||||
def _set_error(self, msg):
|
||||
print(f"[audio] {msg}")
|
||||
with self._lock:
|
||||
self._status["error"] = msg
|
||||
self._status["running"] = False
|
||||
self._running = False
|
||||
|
||||
def _record_beat(self, bpm, beat_type="unknown", beat_type_confidence=0.0):
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
self._status["last_beat_ts"] = now
|
||||
self._status["bpm"] = bpm
|
||||
self._status["beat_type"] = beat_type
|
||||
self._status["beat_type_confidence"] = float(beat_type_confidence or 0.0)
|
||||
self._status["beat_seq"] = int(self._status.get("beat_seq", 0)) + 1
|
||||
try:
|
||||
from util.beat_driver_route import notify_beat_detected
|
||||
|
||||
notify_beat_detected()
|
||||
except Exception as e:
|
||||
print(f"[audio] beat driver route: {e}")
|
||||
|
||||
def _run_loop(self, device):
|
||||
try:
|
||||
import argparse
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
except Exception as e:
|
||||
self._set_error(f"audio deps unavailable: {e}")
|
||||
return
|
||||
|
||||
try:
|
||||
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
beat_detect_path = os.path.join(root_dir, "tests", "beat_detect.py")
|
||||
spec = importlib.util.spec_from_file_location("beat_detect_runtime", beat_detect_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise RuntimeError("cannot load tests/beat_detect.py")
|
||||
beat_mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(beat_mod)
|
||||
|
||||
if device is None:
|
||||
try:
|
||||
device = int(sd.default.device[0])
|
||||
except Exception:
|
||||
device = -1
|
||||
if device is None or device < 0:
|
||||
raise RuntimeError(
|
||||
"no default input device; open Audio, pick an input, then Start"
|
||||
)
|
||||
|
||||
dev_info = sd.query_devices(device, "input")
|
||||
sample_rate = int(dev_info["default_samplerate"])
|
||||
|
||||
args = argparse.Namespace(
|
||||
mode="aubio",
|
||||
device=device,
|
||||
sample_rate=sample_rate,
|
||||
hop_size=256,
|
||||
win_mult=2,
|
||||
min_band_hz=45.0,
|
||||
max_band_hz=180.0,
|
||||
energy_weight=0.7,
|
||||
flux_weight=0.3,
|
||||
threshold_multiplier=1.35,
|
||||
ema_alpha=0.08,
|
||||
min_ioi_ms=85.0,
|
||||
bpm_window=8,
|
||||
post_url="",
|
||||
aubio_method="default",
|
||||
aubio_threshold=0.12,
|
||||
silence_gate_db=-58.0,
|
||||
)
|
||||
runtime = beat_mod.BeatDetectRuntime(args)
|
||||
runtime.setup(sample_rate=sample_rate)
|
||||
hop_size = runtime.frame_size
|
||||
|
||||
audio_q = queue.Queue(maxsize=64)
|
||||
|
||||
def callback(indata, frames, _time_info, status):
|
||||
_ = frames
|
||||
if status:
|
||||
print(f"[audio] status: {status}")
|
||||
mono = np.asarray(indata[:, 0], dtype=np.float32)
|
||||
if not audio_q.full():
|
||||
audio_q.put_nowait(mono)
|
||||
|
||||
stream = sd.InputStream(
|
||||
device=device,
|
||||
channels=1,
|
||||
samplerate=sample_rate,
|
||||
blocksize=hop_size,
|
||||
callback=callback,
|
||||
)
|
||||
with self._lock:
|
||||
self._stream = stream
|
||||
stream.start()
|
||||
try:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
frame = audio_q.get(timeout=0.1)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if frame.shape[0] != hop_size:
|
||||
if frame.shape[0] > hop_size:
|
||||
frame = frame[:hop_size]
|
||||
else:
|
||||
frame = np.pad(frame, (0, hop_size - frame.shape[0]))
|
||||
event = runtime.process_frame(frame, now_s=time.time())
|
||||
if event is None:
|
||||
continue
|
||||
bpm = event.get("bpm")
|
||||
self._record_beat(
|
||||
bpm,
|
||||
beat_type=event.get("beat_type", "unknown"),
|
||||
beat_type_confidence=event.get("beat_type_confidence", 0.0),
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
stream.stop()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
stream.close()
|
||||
except Exception:
|
||||
pass
|
||||
with self._lock:
|
||||
if self._stream is stream:
|
||||
self._stream = None
|
||||
except Exception as e:
|
||||
self._set_error(f"detector failed: {e}")
|
||||
return
|
||||
finally:
|
||||
with self._lock:
|
||||
self._running = False
|
||||
self._status["running"] = False
|
||||
263
src/util/beat_driver_route.py
Normal file
263
src/util/beat_driver_route.py
Normal file
@@ -0,0 +1,263 @@
|
||||
"""Server-side routing of audio beats to LED drivers (no browser required)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict, List, Optional, Set
|
||||
|
||||
_route_lock = threading.Lock()
|
||||
_beat_route: Dict[str, Any] = {
|
||||
"enabled": False,
|
||||
"device_names": [],
|
||||
"wire_preset_id": "2",
|
||||
"is_manual": False,
|
||||
"pattern": "",
|
||||
"manual_beat_n": 1,
|
||||
}
|
||||
_beat_counter: int = 0
|
||||
_main_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
|
||||
def set_beat_route_main_loop(loop: asyncio.AbstractEventLoop) -> None:
|
||||
global _main_loop
|
||||
_main_loop = loop
|
||||
|
||||
|
||||
def update_beat_route(payload: Dict[str, Any]) -> None:
|
||||
"""Internal: set or clear routing from explicit fields (tests / future APIs)."""
|
||||
global _beat_route, _beat_counter
|
||||
if not isinstance(payload, dict):
|
||||
return
|
||||
with _route_lock:
|
||||
if payload.get("enabled") is False:
|
||||
_beat_route = {**_beat_route, "enabled": False}
|
||||
_beat_counter = 0
|
||||
return
|
||||
names = payload.get("device_names")
|
||||
if not isinstance(names, list):
|
||||
names = []
|
||||
try:
|
||||
n_raw = int(payload.get("manual_beat_n", 1))
|
||||
except (TypeError, ValueError):
|
||||
n_raw = 1
|
||||
manual_n = max(1, min(64, n_raw))
|
||||
_beat_route = {
|
||||
"enabled": bool(payload.get("enabled", False)),
|
||||
"device_names": [str(n).strip() for n in names if str(n).strip()],
|
||||
"wire_preset_id": str(payload.get("wire_preset_id") or "2"),
|
||||
"is_manual": bool(payload.get("is_manual", False)),
|
||||
"pattern": str(payload.get("pattern") or "").strip(),
|
||||
"manual_beat_n": manual_n,
|
||||
}
|
||||
_beat_counter = 0
|
||||
|
||||
|
||||
def get_beat_route() -> Dict[str, Any]:
|
||||
with _route_lock:
|
||||
return dict(_beat_route)
|
||||
|
||||
|
||||
def _coerce_manual_beat_n(body: Any) -> int:
|
||||
"""Beats between audio-triggered selects (led-controller only); default 1 = every beat."""
|
||||
if not isinstance(body, dict):
|
||||
return 1
|
||||
raw = body.get("manual_beat_n")
|
||||
if raw is None:
|
||||
return 1
|
||||
try:
|
||||
n = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
return 1
|
||||
return max(1, min(64, n))
|
||||
|
||||
|
||||
def _coerce_auto_from_body(body: Any) -> bool:
|
||||
"""Match JS ``coercePresetAuto`` / ``build_preset_dict`` (default: auto-run)."""
|
||||
if not isinstance(body, dict):
|
||||
return True
|
||||
raw = body.get("auto", body.get("a", True))
|
||||
if isinstance(raw, bool):
|
||||
return raw
|
||||
if raw is None:
|
||||
return True
|
||||
if isinstance(raw, int):
|
||||
return raw != 0
|
||||
if isinstance(raw, str):
|
||||
lowered = raw.strip().lower()
|
||||
if lowered in ("false", "0", "no", "off"):
|
||||
return False
|
||||
if lowered in ("true", "1", "yes", "on"):
|
||||
return True
|
||||
return True
|
||||
|
||||
|
||||
def sync_beat_route_from_push_sequence(sequence: List[Any]) -> None:
|
||||
"""
|
||||
Update beat routing from a ``/presets/push`` body ``sequence`` (list of v1 dicts).
|
||||
|
||||
When the batch includes a ``select`` and preset bodies, and the selected preset is
|
||||
manual (auto off), enables the route; otherwise disables it.
|
||||
"""
|
||||
merged_presets: Dict[str, Any] = {}
|
||||
last_select: Optional[Dict[str, Any]] = None
|
||||
for item in sequence:
|
||||
if isinstance(item, str):
|
||||
try:
|
||||
item = json.loads(item)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not isinstance(item, dict) or item.get("v") != "1":
|
||||
continue
|
||||
pr = item.get("presets")
|
||||
if isinstance(pr, dict):
|
||||
merged_presets.update(pr)
|
||||
sel = item.get("select")
|
||||
if isinstance(sel, dict) and sel:
|
||||
last_select = sel
|
||||
if not last_select:
|
||||
return
|
||||
|
||||
device_names = [str(k).strip() for k in last_select.keys() if str(k).strip()]
|
||||
if not device_names:
|
||||
update_beat_route({"enabled": False})
|
||||
return
|
||||
|
||||
wire_ids: Set[str] = set()
|
||||
for name in device_names:
|
||||
val = last_select.get(name)
|
||||
if isinstance(val, list) and val:
|
||||
wire_ids.add(str(val[0]).strip())
|
||||
elif val is not None:
|
||||
wire_ids.add(str(val).strip())
|
||||
if len(wire_ids) != 1:
|
||||
update_beat_route({"enabled": False})
|
||||
return
|
||||
wire_preset_id = wire_ids.pop()
|
||||
preset_body = merged_presets.get(wire_preset_id)
|
||||
if preset_body is None:
|
||||
for k, v in merged_presets.items():
|
||||
if str(k).strip() == wire_preset_id:
|
||||
preset_body = v
|
||||
break
|
||||
if not isinstance(preset_body, dict):
|
||||
update_beat_route({"enabled": False})
|
||||
return
|
||||
|
||||
if _coerce_auto_from_body(preset_body):
|
||||
update_beat_route({"enabled": False})
|
||||
return
|
||||
|
||||
pattern = str(preset_body.get("pattern") or preset_body.get("p") or "").strip()
|
||||
if pattern and not _pattern_supports_manual(pattern):
|
||||
update_beat_route({"enabled": False})
|
||||
return
|
||||
|
||||
update_beat_route(
|
||||
{
|
||||
"enabled": True,
|
||||
"device_names": device_names,
|
||||
"wire_preset_id": wire_preset_id,
|
||||
"is_manual": True,
|
||||
"pattern": pattern,
|
||||
"manual_beat_n": _coerce_manual_beat_n(preset_body),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _pattern_supports_manual(pattern_key: str) -> bool:
|
||||
if not pattern_key:
|
||||
return True
|
||||
try:
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
root = os.path.abspath(os.path.join(here, "..", ".."))
|
||||
path = os.path.join(root, "db", "pattern.json")
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
meta = data.get(pattern_key)
|
||||
if meta is None:
|
||||
meta = data.get(pattern_key.lower())
|
||||
if not isinstance(meta, dict):
|
||||
return True
|
||||
return meta.get("supports_manual") is not False
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
|
||||
def _macs_for_registry_names(device_names: List[str]) -> List[str]:
|
||||
from models.device import Device
|
||||
|
||||
want = {str(n).strip() for n in device_names if str(n).strip()}
|
||||
if not want:
|
||||
return []
|
||||
devices = Device()
|
||||
macs: List[str] = []
|
||||
seen = set()
|
||||
for did in devices.list():
|
||||
doc = devices.read(did) or {}
|
||||
nm = str(doc.get("name") or "").strip()
|
||||
if nm not in want:
|
||||
continue
|
||||
key = str(did).strip().lower().replace(":", "").replace("-", "")
|
||||
if len(key) == 12 and key not in seen:
|
||||
seen.add(key)
|
||||
macs.append(key)
|
||||
return macs
|
||||
|
||||
|
||||
async def _deliver_select(device_names: List[str], wire_preset_id: str) -> None:
|
||||
from models.device import Device
|
||||
from models.transport import get_current_sender
|
||||
from util.driver_delivery import deliver_json_messages
|
||||
|
||||
sender = get_current_sender()
|
||||
if not sender:
|
||||
return
|
||||
select = {str(n).strip(): [wire_preset_id] for n in device_names if str(n).strip()}
|
||||
if not select:
|
||||
return
|
||||
msg = json.dumps({"v": "1", "select": select}, separators=(",", ":"))
|
||||
macs = _macs_for_registry_names(list(select.keys()))
|
||||
if not macs:
|
||||
return
|
||||
devices = Device()
|
||||
try:
|
||||
await deliver_json_messages(sender, [msg], macs, devices, delay_s=0.05)
|
||||
except Exception as e:
|
||||
print(f"[beat-route] deliver failed: {e}")
|
||||
|
||||
|
||||
def notify_beat_detected() -> None:
|
||||
"""Invoked from the audio thread when a beat is detected."""
|
||||
global _beat_counter
|
||||
with _route_lock:
|
||||
r = dict(_beat_route)
|
||||
if not r.get("enabled"):
|
||||
return
|
||||
if not r.get("is_manual"):
|
||||
return
|
||||
pattern = r.get("pattern") or ""
|
||||
if pattern and not _pattern_supports_manual(pattern):
|
||||
return
|
||||
names = r.get("device_names") or []
|
||||
if not names:
|
||||
return
|
||||
try:
|
||||
n = int(r.get("manual_beat_n") or 1)
|
||||
except (TypeError, ValueError):
|
||||
n = 1
|
||||
n = max(1, min(64, n))
|
||||
_beat_counter += 1
|
||||
if ((_beat_counter - 1) % n) != 0:
|
||||
return
|
||||
preset_id = str(r.get("wire_preset_id") or "2")
|
||||
names_copy = list(names)
|
||||
loop = _main_loop
|
||||
if loop is None:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(_deliver_select(names_copy, preset_id), loop)
|
||||
except Exception as e:
|
||||
print(f"[beat-route] schedule failed: {e}")
|
||||
@@ -119,13 +119,40 @@ def build_preset_dict(preset_data):
|
||||
else:
|
||||
colors = ["#FFFFFF"]
|
||||
|
||||
def _coerce_auto(raw):
|
||||
if isinstance(raw, bool):
|
||||
return raw
|
||||
if raw is None:
|
||||
return True
|
||||
if isinstance(raw, int):
|
||||
return raw != 0
|
||||
if isinstance(raw, str):
|
||||
lowered = raw.strip().lower()
|
||||
if lowered in ("false", "0", "no", "off"):
|
||||
return False
|
||||
if lowered in ("true", "1", "yes", "on"):
|
||||
return True
|
||||
return True
|
||||
|
||||
auto_raw = preset_data.get("auto", preset_data.get("a", True))
|
||||
auto_bool = _coerce_auto(auto_raw)
|
||||
|
||||
bg_raw = preset_data.get("background", preset_data.get("bg", "#000000"))
|
||||
if isinstance(bg_raw, str):
|
||||
bg = bg_raw if bg_raw.startswith("#") else f"#{bg_raw}"
|
||||
elif isinstance(bg_raw, (list, tuple)) and len(bg_raw) == 3:
|
||||
bg = f"#{int(bg_raw[0]):02x}{int(bg_raw[1]):02x}{int(bg_raw[2]):02x}"
|
||||
else:
|
||||
bg = "#000000"
|
||||
|
||||
# Build payload using the short keys expected by led-driver
|
||||
preset = {
|
||||
"p": preset_data.get("pattern", preset_data.get("p", "off")),
|
||||
"c": colors,
|
||||
"d": preset_data.get("delay", preset_data.get("d", 100)),
|
||||
"b": preset_data.get("brightness", preset_data.get("b", preset_data.get("br", 127))),
|
||||
"a": preset_data.get("auto", preset_data.get("a", True)),
|
||||
"a": auto_bool,
|
||||
"bg": bg,
|
||||
"n1": preset_data.get("n1", 0),
|
||||
"n2": preset_data.get("n2", 0),
|
||||
"n3": preset_data.get("n3", 0),
|
||||
|
||||
Reference in New Issue
Block a user