feat(v2): ESP32-S3 controller with Ethernet, I2S mic beat detection and ESP-NOW

New ESP-IDF project in v2/ that replaces the Pi and the UART bridge:

- W5500 Ethernet, INMP441 beat/tempo detector (from esp32-eth), ESP-NOW master
- LittleFS data store and a JSON API over the Pi's db/*.json collections,
  with /db import and export; web UI minified and gzipped at build time
- Zone play/stop as 0x4C GROUP_CMD, preset push/send as CMD (binary v2
  envelope, byte-identical to src/util/binary_envelope.py), sequencer and
  scenes, driver registration from the boot hello
- Reliable bulk link: acks with retransmit, fragments, state-version digest
  and resync; beat frames sent twice with a lead so drivers fire together
- Bench tooling: lossy-link mode, synthetic beats with echo measurement,
  USB and logic-analyzer latency scripts, and docs/BENCH_TEST.md

Needs the matching led-driver changes (p2p branch) to receive the link and
beat frames.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
jimmyandClaude Sonnet 5.5 committed 2026-10-27 17:15:28 +13:00
1 parent 7336c7fb22
commit a94d6a7de5
37 files changed
+5109

No files matched your search

+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Beat latency from a logic analyzer capture.
Reads a CSV with a time column and two digital channels: the controller beat
marker (CONFIG_BEAT_TEST_GPIO, toggles when the first copy of a beat goes out)
and the driver beat marker (`beat_pin`, toggles when the driver fires). For
every controller edge it finds the next driver edge and reports the delay minus
the deliberate lead (CONFIG_BEAT_LEAD_MS, default 30 ms).
Works with Saleae ("Time [s],Channel 0,Channel 1") and sigrok CSV exports, or
any CSV whose first column is time and whose other columns are 0/1 levels.
analyze_edges.py capture.csv --ctrl 0 --drv 1
"""
import argparse
import csv
import statistics
import sys
def read(path, time_scale):
times, rows = [], []
with open(path, newline="") as f:
for row in csv.reader(f):
if not row or row[0].lstrip().startswith((";", "#")):
continue
try:
t = float(row[0]) * time_scale
levels = [int(float(c)) for c in row[1:]]
except ValueError:
continue # header line
times.append(t)
rows.append(levels)
return times, rows
def edges(times, rows, ch):
out, prev = [], None
for t, r in zip(times, rows):
v = r[ch]
if prev is not None and v != prev:
out.append(t)
prev = v
return out
def analyze(ctrl, drv, lead_s, window_s):
deltas, missed, j = [], 0, 0
for i, t in enumerate(ctrl):
nxt = ctrl[i + 1] if i + 1 < len(ctrl) else float("inf")
while j < len(drv) and drv[j] < t:
j += 1
if j < len(drv) and drv[j] - t <= min(window_s, nxt - t):
deltas.append(drv[j] - t - lead_s)
j += 1
else:
missed += 1
return deltas, missed
def pct(sorted_vals, p):
if not sorted_vals:
return float("nan")
k = min(len(sorted_vals) - 1, int(round(p / 100 * (len(sorted_vals) - 1))))
return sorted_vals[k]
def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("csv")
ap.add_argument("--ctrl", type=int, default=0, help="column index of the controller marker (0 = first channel)")
ap.add_argument("--drv", type=int, default=1, help="column index of the driver marker")
ap.add_argument("--lead-ms", type=float, default=30.0, help="deliberate lead to subtract (CONFIG_BEAT_LEAD_MS)")
ap.add_argument("--window-ms", type=float, default=200.0, help="longest delay still counted as the same beat")
ap.add_argument("--time-unit", choices=("s", "ms", "us", "ns"), default="s", help="unit of the time column")
a = ap.parse_args(argv)
scale = {"s": 1.0, "ms": 1e-3, "us": 1e-6, "ns": 1e-9}[a.time_unit]
times, rows = read(a.csv, scale)
if not times:
sys.exit("no samples read")
ctrl, drv = edges(times, rows, a.ctrl), edges(times, rows, a.drv)
deltas, missed = analyze(ctrl, drv, a.lead_ms / 1000, a.window_ms / 1000)
ms = sorted(d * 1000 for d in deltas)
print(f"controller edges: {len(ctrl)} driver edges: {len(drv)} matched: {len(ms)} unmatched: {missed}")
if not ms:
sys.exit("no matched edges: check channel numbers and --time-unit")
print(f"delay beyond the {a.lead_ms:g} ms lead (ms): min {ms[0]:.2f} mean {statistics.mean(ms):.2f} "
f"median {pct(ms, 50):.2f} p95 {pct(ms, 95):.2f} max {ms[-1]:.2f}")
if len(ms) > 1:
print(f"jitter (std dev): {statistics.stdev(ms):.2f} ms")
if a.lead_ms and ms[0] < -1:
print("note: negative delays mean the lead is larger than the real one; check --lead-ms", file=sys.stderr)
if __name__ == "__main__":
main()
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Minify and gzip the web UI into v2/www_dist for the `www` LittleFS partition.
# Usage: tools/build_web.sh (needs node/npx; WWW_MAX = partition size in bytes)
set -euo pipefail
cd "$(dirname "$0")/.."
SRC=../src
OUT=www_dist
MAX=${WWW_MAX:-1048576}
rm -rf "$OUT"; mkdir -p "$OUT/static"
# Legacy files the v2 UI does not load
SKIP=" app.js main.js rgb-slider.js styles.css "
for f in "$SRC"/static/*; do
[[ -f $f ]] || continue
b=$(basename "$f"); [[ $SKIP == *" $b "* ]] && continue
case $b in
*.min.js) cp "$f" "$OUT/static/$b" ;;
*.js|*.css) npx --yes esbuild@0.25.0 "$f" --minify --target=es2019 --legal-comments=none \
--outfile="$OUT/static/$b" --log-level=warning ;;
*) cp "$f" "$OUT/static/$b" ;;
esac
done
for p in index settings; do
npx --yes html-minifier-terser@7.2.0 --collapse-whitespace --remove-comments \
--minify-css true --minify-js true -o "$OUT/$p.html" "$SRC/templates/$p.html"
done
# Ship only the .gz copies; the server sends Content-Encoding: gzip
find "$OUT" -type f -exec gzip -9 -n -f {} \;
find "$OUT" -name '*.gz' -exec gzip -t {} \;
total=$(find "$OUT" -name '*.gz' -printf '%s\n' | paste -sd+ | bc)
echo "www_dist: $(find "$OUT" -name '*.gz' | wc -l) files, $total bytes (limit $MAX)"
(( total <= MAX )) || { echo "ERROR: web UI exceeds www partition" >&2; exit 1; }
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env python3
"""Beat latency over USB, no logic analyzer.
Starts the controller's synthetic beat generator, reads one timestamped line
from each board's USB serial port, and works out how long the driver took to
fire after the controller sent each beat.
controller prints BT <beat_idx> <esp_timer_us> (just before the first copy goes out)
driver prints FIRE <beat_idx> <ticks_us> (when the beat triggers a pattern step)
The two boards have unrelated clocks and USB adds a variable delay of its own.
Each board's clock is mapped onto the PC clock with the *fastest* USB deliveries
(lower envelope of host-time minus board-time, fitted per 5 s window, so crystal
drift is removed). Differences between the two mapped clocks are then the latency.
What remains is the gap between the two USB paths' minimum delays (about a
millisecond at worst) plus noise; the report prints the residual USB jitter so
you can judge it. The driver needs "usb_log": true in settings.json.
usb_latency.py --ctrl-port /dev/ttyACM0 --drv-port /dev/ttyACM1 --url http://10.1.1.180
"""
import argparse
import json
import re
import statistics
import threading
import time
import urllib.request
import serial
TICKS_PERIOD_US = 1 << 30 # MicroPython ticks_us wraps here
class Reader(threading.Thread):
def __init__(self, port, pattern):
super().__init__(daemon=True)
self.ser = serial.Serial(port, 115200, timeout=0.05)
self.pattern = re.compile(pattern)
self.events = [] # (idx, device_us, host_s)
self.running = True
self.buf = b""
def run(self):
while self.running:
try:
chunk = self.ser.read(4096)
except serial.SerialException:
break
if not chunk:
continue
now = time.perf_counter() # one stamp per USB chunk
self.buf += chunk
*lines, self.buf = self.buf.split(b"\n")
for ln in lines:
m = self.pattern.search(ln.decode(errors="replace"))
if m:
self.events.append((int(m.group(1)), int(m.group(2)), now))
def unwrap(events, period=None):
"""Make device timestamps monotonic (handles the driver's wrapping tick counter)."""
out, add, prev = [], 0, None
for idx, us, host in events:
if period and prev is not None and us + add < prev - period // 2:
add += period
out.append((idx, (us + add) / 1e6, host))
prev = us + add
return out
def fit_clock(events, window_s=5.0):
"""Map device seconds to host seconds using the fastest deliveries. Returns (a, b, residuals)."""
pts = [(d, h - d) for _, d, h in events] # (device time, host minus device)
t0 = pts[0][0]
mins = {}
for d, off in pts:
k = int((d - t0) // window_s)
if k not in mins or off < mins[k][1]:
mins[k] = (d, off)
xs, ys = zip(*mins.values()) if len(mins) > 1 else (zip(*mins.values()))
xs, ys = list(xs), list(ys)
if len(xs) < 2:
b, a = 0.0, ys[0]
else:
mx, my = statistics.mean(xs), statistics.mean(ys)
den = sum((x - mx) ** 2 for x in xs) or 1.0
b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / den
a = my - b * mx
lowest = min(off - (a + b * d) for d, off in pts)
a += lowest # shift so the fastest delivery sits exactly on the line
resid = sorted((off - (a + b * d)) * 1000 for d, off in pts)
return a, b, resid
def pct(vals, p):
return vals[min(len(vals) - 1, int(round(p / 100 * (len(vals) - 1))))]
def main():
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--ctrl-port", default="/dev/ttyACM0")
ap.add_argument("--drv-port", default="/dev/ttyACM1")
ap.add_argument("--url", default="http://10.1.1.180")
ap.add_argument("--bpm", type=float, default=120)
ap.add_argument("--beats", type=int, default=200)
ap.add_argument("--lead-ms", type=float, default=30.0, help="CONFIG_BEAT_LEAD_MS")
ap.add_argument("--no-start", action="store_true", help="do not start the generator (it is already running)")
a = ap.parse_args()
rc = Reader(a.ctrl_port, r"\bBT (\d+) (\d+)")
rd = Reader(a.drv_port, r"\bFIRE (\d+) (\d+)")
rc.start(); rd.start()
time.sleep(0.5)
if not a.no_start:
req = urllib.request.Request(a.url + "/test/beats", data=json.dumps({"bpm": a.bpm, "count": a.beats}).encode(),
method="POST")
urllib.request.urlopen(req, timeout=10).read()
wait = a.beats * 60.0 / a.bpm + 4
print(f"running {a.beats} beats at {a.bpm:g} BPM, about {wait:.0f} s ...")
time.sleep(wait)
rc.running = rd.running = False
time.sleep(0.2)
ctrl, drv = unwrap(rc.events), unwrap(rd.events, TICKS_PERIOD_US)
if len(ctrl) < 5 or len(drv) < 5:
raise SystemExit(f"too few events: controller {len(ctrl)}, driver {len(drv)} "
"(driver needs usb_log:true; check ports)")
ca, cb, cres = fit_clock(ctrl)
da, db, dres = fit_clock(drv)
c_host = {i: d + ca + cb * d for i, d, _ in ctrl}
lat = []
for i, d, _ in drv:
if i in c_host:
lat.append((d + da + db * d - c_host[i]) * 1000 - a.lead_ms)
lat.sort()
print(f"controller events {len(ctrl)}, driver events {len(drv)}, matched {len(lat)}")
print(f"clock drift: controller {cb*1e6:+.1f} ppm, driver {db*1e6:+.1f} ppm (relative to the PC)")
print(f"USB delivery jitter above the fastest (ms), p50/p95/max: controller {pct(cres,50):.2f}/{pct(cres,95):.2f}/{cres[-1]:.2f} "
f"driver {pct(dres,50):.2f}/{pct(dres,95):.2f}/{dres[-1]:.2f}")
if not lat:
raise SystemExit("no beat index matched on both ports")
print(f"delay beyond the {a.lead_ms:g} ms lead (ms): min {lat[0]:.2f} mean {statistics.mean(lat):.2f} "
f"median {pct(lat,50):.2f} p95 {pct(lat,95):.2f} max {lat[-1]:.2f} std {statistics.pstdev(lat):.2f}")
print("accuracy: systematic error about +/-1 ms (difference of the two USB paths' fastest delivery); "
"use the logic analyzer if you need better")
if __name__ == "__main__":
main()