From 495da4e366b4ddef75a79e4eae0cec9fd6462013 Mon Sep 17 00:00:00 2001 From: jimmy Date: Mon, 2 Nov 2026 23:41:17 +1300 Subject: [PATCH] feat(espnow): app-file OTA over the reliable link New espnow_ota module, loaded on the first OTA packet (types 0x07-0x0C, status reply 0x26). Files are written by chunk index to .new, checked against their sha256, and moved into place together on commit, followed by a reset. Unchanged files are skipped, and a repeated commit after the reset still gets OK. - receive ESP-NOW frames from esp.irq() so frames are not lost while presets.tick() runs; polling remains as a fallback - raise rxbuf to 2048 and the link inbox to 24 frames - file_hashes: MicroPython's sha256 has no hexdigest() - bench stats line includes esp.stats() Co-Authored-By: Claude Opus 5.5 --- src/espnow_ota.py | 293 +++++++++++++++++++++++++++++++++++++++ src/espnow_rx.py | 3 +- src/espnow_transport.py | 19 ++- src/espnow_wire.py | 2 + src/file_hashes.py | 5 +- src/main.py | 35 +++-- tests/test_espnow_ota.py | 231 ++++++++++++++++++++++++++++++ tests/test_espnow_rx.py | 4 +- 8 files changed, 577 insertions(+), 15 deletions(-) create mode 100644 src/espnow_ota.py create mode 100644 tests/test_espnow_ota.py diff --git a/src/espnow_ota.py b/src/espnow_ota.py new file mode 100644 index 0000000..591ef41 --- /dev/null +++ b/src/espnow_ota.py @@ -0,0 +1,293 @@ +"""App-file OTA over ESP-NOW (driver side). Imported on the first OTA packet only. + +Inner packets, carried in reliable-link DATA frames (unicast acked, or broadcast): + 0x07 BEGIN [4C][07][sess u16][size u32][sha256 32B][plen][path] + 0x08 DATA [4C][08][sess u16][chunk u16] + up to CHUNK bytes at chunk * CHUNK + 0x09 END [4C][09][sess u16] hash the staged file + 0x0A COMMIT [4C][0A][sess u16] move every staged file into place, then reset + 0x0B ABORT [4C][0B][sess u16] delete the staged files + 0x0C MISSING [4C][0C][sess u16] report which chunks of the current file are missing +Reply, sent straight back (not through the link): + 0x26 STATUS [4C][26][sess u16][op][code] + arg + arg: [cnt u16] + missing bitmap for MISSING, else received bytes u32 +Files are written to .new; a new session discards whatever an older one staged. +The committed session is kept in ota_last: the reset can beat a lost reply, and the +controller then asks again. +""" +import struct + +import file_hashes + +WIRE_MAGIC = 0x4C +OP_BEGIN = 0x07 +OP_DATA = 0x08 +OP_END = 0x09 +OP_COMMIT = 0x0A +OP_ABORT = 0x0B +OP_MISSING = 0x0C +MSG_STATUS = 0x26 + +OK = 0 +READY = 1 +SAME = 2 +BADHASH = 3 +INCOMPLETE = 4 +BADPATH = 5 +IOERR = 6 +NOSESS = 7 +TOOBIG = 8 + +CHUNK = 944 # 4 link fragments of 238 bytes minus this packet's 8-byte header +MAX_SIZE = 65536 +STAGED_SUFFIX = ".new" +LAST_FILE = "ota_last" # session last committed, so a repeated COMMIT after the reset still gets OK + +try: + import uos as os +except ImportError: + import os + + +def safe_path(path): + """Relative path to a root file, patterns/... or lib/..., never the hash manifest.""" + if not path or path.startswith("/") or "\\" in path or path == file_hashes.MANIFEST_FILENAME: + return False + parts = path.split("/") + if any(p in ("", ".", "..") for p in parts): + return False + return len(parts) == 1 or parts[0] in ("patterns", "lib") + + +def _makedirs(path): + parts = path.split("/")[:-1] + for i in range(1, len(parts) + 1): + try: + os.mkdir("/".join(parts[:i])) + except OSError: + pass + + +def _remove(path): + try: + os.remove(path) + except OSError: + pass + + +class Ota: + def __init__(self, send, reset=None): + self.send = send # send(mac, frame) + self.reset = reset # called after a commit; defaults to machine.reset + self.sess = 0 + self.staged = {} # path -> sha hex of a verified .new file + self.cur = None # [path, size, sha hex, file, have bitmap (bytearray), received bytes] + try: + with open(LAST_FILE) as f: + self.committed = int(f.read()) + except (OSError, ValueError): + self.committed = 0 + + def handle(self, mac, pkt): + if len(pkt) < 4: + return + op = pkt[1] + sess = pkt[2] | (pkt[3] << 8) + if sess == 0: + return + if op == OP_COMMIT and sess == self.committed: + self._reply(mac, sess, op, OK, struct.pack(" MAX_SIZE: + self._reply(mac, sess, OP_BEGIN, TOOBIG) + return + if self.cur is not None: + if self.cur[0] == path and self.cur[2] == sha: + self._reply(mac, sess, OP_BEGIN, READY) # our earlier reply was lost + return + self._close_cur(True) + if self.staged.get(path) == sha: + self._reply(mac, sess, OP_BEGIN, READY) + return + try: + if file_hashes._hash_file(path) == sha: + self._reply(mac, sess, OP_BEGIN, SAME) + return + except OSError: + pass # no such file yet + try: + _makedirs(path) + f = open(path + STAGED_SUFFIX, "w+b") + except OSError: + self._reply(mac, sess, OP_BEGIN, IOERR) + return + cnt = (size + CHUNK - 1) // CHUNK + self.staged.pop(path, None) + self.cur = [path, size, sha, f, bytearray((cnt + 7) // 8), 0] + self._reply(mac, sess, OP_BEGIN, READY) + + def _data(self, pkt): + cur = self.cur + if cur is None or len(pkt) < 6: + return + idx = pkt[4] | (pkt[5] << 8) + body = pkt[6:] + off = idx * CHUNK + have = cur[4] + if off >= cur[1] or have[idx >> 3] & (1 << (idx & 7)): + return # past the end, or a copy we already wrote + if len(body) != min(CHUNK, cur[1] - off): + return + f = cur[3] + try: + end = f.seek(0, 2) + if end < off: + f.write(bytes(off - end)) # a chunk before this one is still missing + f.seek(off) + f.write(body) + except OSError: + self._close_cur(True) + return + have[idx >> 3] |= 1 << (idx & 7) + cur[5] += len(body) + + def _missing_bitmap(self): + cur = self.cur + cnt = (cur[1] + CHUNK - 1) // CHUNK + miss = bytearray(len(cur[4])) + for i in range(cnt): + if not cur[4][i >> 3] & (1 << (i & 7)): + miss[i >> 3] |= 1 << (i & 7) + return cnt, miss + + def _missing(self, mac): + if self.cur is None: + self._reply(mac, self.sess, OP_MISSING, OK, struct.pack(" controller 0x24 ECHO [4C][24][beat_idx u16] driver -> controller, bench only 0x25 BEAT [4C][25][flags][idx u16][period u16][fire_in] handled by BeatSync + 0x26 OTA [4C][26][sess u16][op][code] + arg driver -> controller, see espnow_ota.py Everything else (announce, groups, cmd, group cmd, plain JSON) goes straight to the handler. """ import struct @@ -25,7 +26,7 @@ F_ACK = 1 F_COMMIT = 2 HDR = 9 -INBOX_MAX = 16 +INBOX_MAX = 24 DONE_MAX = 16 ASM_TIMEOUT_MS = 3000 RESYNC_MIN_GAP_MS = 3000 diff --git a/src/espnow_transport.py b/src/espnow_transport.py index 69e24c3..a2045b4 100644 --- a/src/espnow_transport.py +++ b/src/espnow_transport.py @@ -14,6 +14,8 @@ from espnow_wire import ( MSG_CMD, MSG_GROUP_CMD, MSG_GROUPS, + MSG_OTA_FIRST, + MSG_OTA_LAST, MSG_PING_REQ, cmd_envelope, pack_announce, @@ -31,6 +33,7 @@ _PING_DELAY_MS_MIN = 50 _PING_DELAY_MS_MAX = 500 _esp = None +_ota_send = None _groups_received = False _debug = False @@ -56,6 +59,11 @@ def init_espnow(settings): except Exception as e: print("espnow sta channel set failed:", e) _esp = espnow.ESPNow() + try: + # The default 526 bytes holds two full frames; an OTA chunk is four + _esp.config(rxbuf=int(settings.get("espnow_rxbuf", 2048))) + except Exception as e: + print("espnow rxbuf set failed:", e) _esp.active(True) try: _esp.add_peer(BROADCAST_MAC) @@ -125,8 +133,17 @@ def send_boot_announce(settings): def _handle_packet(host, pkt, settings, presets): - global _groups_received + global _groups_received, _ota_send mt = wire_msg_type(pkt) + if mt is not None and MSG_OTA_FIRST <= mt <= MSG_OTA_LAST: + import espnow_ota + + if _ota_send is None: + from espnow_rx import make_sender + + _ota_send = make_sender(_esp) + espnow_ota.handle(host, pkt, _ota_send) + return if mt == MSG_GROUPS: ids = parse_groups(pkt) if ids is not None: diff --git a/src/espnow_wire.py b/src/espnow_wire.py index 09ef807..8e36e79 100644 --- a/src/espnow_wire.py +++ b/src/espnow_wire.py @@ -11,6 +11,8 @@ MSG_CMD = 0x03 MSG_GROUP_CMD = 0x04 MSG_PING_REQ = 0x05 MSG_PING_RSP = 0x06 +MSG_OTA_FIRST = 0x07 # 0x07..0x0C app-file OTA, see espnow_ota.py +MSG_OTA_LAST = 0x0C BROADCAST_MAC = b"\xff\xff\xff\xff\xff\xff" diff --git a/src/file_hashes.py b/src/file_hashes.py index e7fca81..4afe59b 100644 --- a/src/file_hashes.py +++ b/src/file_hashes.py @@ -47,16 +47,17 @@ def save(files): def _hash_file(path): + import binascii import hashlib h = hashlib.sha256() with open(path, "rb") as f: while True: - chunk = f.read(256) + chunk = f.read(1024) if not chunk: break h.update(chunk) - return h.hexdigest() + return binascii.hexlify(h.digest()).decode() # MicroPython's sha256 has no hexdigest() def _walk_dir(base, prefix, out): diff --git a/src/main.py b/src/main.py index 465399f..6a5c4bf 100644 --- a/src/main.py +++ b/src/main.py @@ -77,16 +77,33 @@ beat_pin = None if settings.get("beat_pin") is not None: beat_pin = machine.Pin(int(settings["beat_pin"]), machine.Pin.OUT) +def _drain(e): + # Runs scheduled between bytecodes, so frames are taken while presets.tick() is still busy. + # irecv reuses its buffers: on_packet handles beats in place and copies everything else. + while True: + try: + host, msg = e.irecv(0) + except OSError: + return + if not host: + return + if msg: + try: + link.on_packet(host, msg, utime.ticks_ms()) + except Exception as ex: + print("espnow rx error:", ex) + + +try: + esp.irq(_drain) + use_irq = True +except AttributeError: # firmware built without ESP-NOW irq support: poll as before + use_irq = False + while True: wdt.feed() - while esp.any(): - host, msg = esp.recv(0) - if not host or not msg: - continue - try: - link.on_packet(host, msg, utime.ticks_ms()) - except Exception as e: - print("espnow rx error:", e) + if not use_irq: + _drain(esp) now = utime.ticks_ms() try: link.poll(now) @@ -103,7 +120,7 @@ while True: link.echo(beat_sync.last_idx) # only real triggers: an invented beat has no send time to compare if bench and utime.ticks_diff(now, last_stat) >= 5000: last_stat = now - print("beat", beat_sync.stats, "link", link.stats) + print("beat", beat_sync.stats, "link", link.stats, "radio", esp.stats()) # radio: tx, tx_resp, tx_fail, rx, rx_dropped presets.tick() if fire_log is not None: print("FIRE", fire_log[0], fire_log[1]) diff --git a/tests/test_espnow_ota.py b/tests/test_espnow_ota.py new file mode 100644 index 0000000..912f1e9 --- /dev/null +++ b/tests/test_espnow_ota.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Ota: staging, chunk bitmap, hash checks, commit and sessions. Runs in a temp directory.""" +import hashlib +import json +import os +import struct +import tempfile + +import espnow_ota as o +import file_hashes + +M = b"\x01\x02\x03\x04\x05\x06" + + +def begin(sess, path, body): + p = path.encode() + return struct.pack("> 3)] & (1 << (i & 7)): + r(chunk(11, body, i)) + assert r(cmd(o.OP_END, 11))[1] == o.OK + r(cmd(o.OP_COMMIT, 11)) + assert open("lib/x.py", "rb").read() == body + os.chdir(old) + + +@in_tmp +def test_commit_repeated_after_reset(): + r = Rig() + r.send_file(14, "a.py", BODY) + assert r(cmd(o.OP_COMMIT, 14))[1] == o.OK + fresh = Rig() # the driver reset before its reply arrived + op, code, arg = fresh(cmd(o.OP_COMMIT, 14)) + assert code == o.OK and fresh.resets == 0 + assert fresh(cmd(o.OP_COMMIT, 15))[1] == o.NOSESS + + +@in_tmp +def test_empty_file(): + r = Rig() + assert r.send_file(13, "lib/pkg/__init__.py", b"") == o.OK + r(cmd(o.OP_COMMIT, 13)) + assert open("lib/pkg/__init__.py", "rb").read() == b"" + + +@in_tmp +def test_too_big_rejected(): + r = Rig() + pkt = bytearray(begin(12, "a.py", b"x")) + struct.pack_into("