3 Commits
Author SHA1 Message Date
jimmyandCursor 1edcb8b1f7 feat(cli): skip unchanged files using device file_hashes.json
Read and update file_hashes.json on deploy; add --force-upload and tests.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-17 19:14:51 +12:00
jimmyandCursor ccc215acbd fix(cli): lazy settings fetch and full device erase
Download settings only for --show or when applying edits; skip upload
when unchanged; erase entire device root including settings.json.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-16 21:14:54 +12:00
jimmyandCursor 580fd11aca fix(cli): skip redundant settings uploads when unchanged
Only upload settings when edited values differ from current device settings to avoid unnecessary resets.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-05-04 22:48:54 +12:00
6 changed files with 287 additions and 170 deletions

No files matched your search

+5 -5
View File
@@ -25,14 +25,14 @@ Connection is always via **`-p` / `--port`** (default `/dev/ttyACM0`). There is
| `-r`, `--reset` | Reset the device |
| `-f`, `--follow` | Follow serial output (optional timeout seconds) |
| `--pause` | Sleep N seconds (for chained actions) |
| `-u`, `--upload` | Recursive upload: `-u SRC [DEST]` |
| `--src`, `--lib` | Upload `src/` or a tree to `/lib` |
| `-e` | Erase device code (keeps `settings.json`) |
| `-u`, `--upload` | Recursive upload: `-u SRC [DEST]` (skips unchanged files via `file_hashes.json` on device) |
| `--src`, `--lib`, `--all` | Deploy led-driver trees to flash root, `patterns/`, and `lib/` |
| `--force-upload` | Upload every file; ignore `file_hashes.json` |
| `-e`, `--erase` | Erase everything at device root (including `settings.json` and `presets.json`) |
| `--rm` | Remove a path on the device |
| `--ota` | Update a v2 controller over Ethernet: `--ota HOST [IMAGE]` (no serial port; image defaults to `led_controller_v2.bin` in `./v2/build`, `./build` or `../v2/build`) |
| `--flash` | Flash a firmware binary (uses **esptool** on the host) |
**Default behaviour:** the tool always downloads `settings.json`, prints the merged view, and uploads again **only** when you pass setting edits (`--name`, `--leds`, …), **`--preset`**, or the relevant upload/flash/erase actions in order.
**Settings I/O:** With no arguments, **`led-cli`** prints help (no device access). **`--show`** downloads `settings.json` and prints it (read-only). Setting fields (`--name`, `-b`, …) download once, merge, print, and **upload only when a value actually changed**. **`--preset`** alone only touches **`presets.json`** (no `settings.json` download). Ordered actions (`--reset`, `--upload`, `-e`, …) run without touching settings unless you also pass **`--show`** or setting / preset edits as above.
Run **`python cli.py -h`** for the full epilog and argument list.
+126 -83
View File
@@ -2,8 +2,10 @@
"""
LED Bar Configuration CLI Tool
Command-line interface for downloading, editing, and uploading settings.json
to/from MicroPython devices via mpremote.
Command-line interface for editing settings.json on MicroPython devices via mpremote.
Settings are downloaded from the device only when displaying (--show) or when applying
setting changes; uploads occur only when something changed. Use --erase to clear the
device filesystem (including settings).
"""
import json
@@ -16,7 +18,6 @@ from typing import Dict, Any, List, Optional
import tempfile
import os
from ota import ota_update
from device import copy_file, DeviceConnection, firmware_default_device_name
@@ -149,14 +150,14 @@ _FLAGS_WITH_VALUE = frozenset({
'-p', '--port', '-n', '--name', '--pin', '-b', '--brightness',
'-l', '--leds', '-d', '-debug', '--debug', '-o', '--order',
'--preset', '--pattern', '--default', '--transport', '--ssid',
'--wifi-password', '--wifi-channel', '--ota', '--src', '--lib', '--patterns', '--paterns',
'--wifi-password', '--wifi-channel', '--src', '--lib', '--patterns', '--paterns',
})
def _get_ordered_actions(argv: List[str]) -> List[tuple]:
"""
Scan argv and return list of (action_name, value) in order of appearance.
Actions: flash, ota, pause, reset, upload, ls, erase_all, rm, follow.
Actions: flash, pause, reset, upload, ls, erase_all, rm, follow.
"""
actions = []
i = 1
@@ -180,16 +181,6 @@ def _get_ordered_actions(argv: List[str]) -> List[tuple]:
else:
raise ValueError("--flash requires an argument")
continue
if arg == '--ota':
vals = []
i += 1
while i < len(argv) and not argv[i].startswith('-') and len(vals) < 2:
vals.append(argv[i])
i += 1
if not vals:
raise ValueError("--ota requires a host (and optionally an image path)")
actions.append(('ota', vals))
continue
if arg in ('-r', '--reset'):
actions.append(('reset', None))
i += 1
@@ -232,6 +223,10 @@ def _get_ordered_actions(argv: List[str]) -> List[tuple]:
actions.append(('upload', ["lib", "lib"]))
i += 1
continue
if arg == '--force-upload':
# Handled via argparse; skip during action scan
i += 1
continue
if arg == '--lib':
# Upload local DIR (default: ./lib) to /lib on device
local_dir = "lib"
@@ -290,10 +285,13 @@ def main() -> None:
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Download and print current settings
# Show help (no device I/O)
%(prog)s
# Download, edit device name and brightness, then upload
# Show current settings.json from device (read-only)
%(prog)s --show
# Edit device name and brightness, then upload only if values changed
%(prog)s -n "LED-Strip-1" -b 128
# Set multiple parameters
@@ -315,10 +313,6 @@ Examples:
%(prog)s --flash firmware.bin
%(prog)s -p /dev/ttyUSB0 --flash /path/to/firmware.bin
# Update a v2 controller over Ethernet (image defaults to led_controller_v2.bin under ./v2/build, ./build or ../v2/build)
%(prog)s --ota 10.1.1.180
%(prog)s --ota 10.1.1.180 /path/to/led_controller_v2.bin
# Reset, pause 2 seconds, then follow output
%(prog)s --reset --pause 2 --follow
@@ -496,6 +490,12 @@ Examples:
help="Upload ./src (excluding patterns), ./src/patterns, and ./lib."
)
parser.add_argument(
"--force-upload",
action="store_true",
help="Upload every file; ignore file_hashes.json on the device",
)
parser.add_argument(
"--patterns", "--paterns",
dest="patterns_dir",
@@ -515,7 +515,7 @@ Examples:
"-e", "--erase",
dest="erase_all",
action="store_true",
help="Erase all code on the device (delete all files except settings.json)"
help="Erase the device filesystem: delete all files and directories at device root (including settings.json and presets.json)",
)
parser.add_argument(
@@ -524,13 +524,6 @@ Examples:
help="Delete a file or directory on the device (recursive for directories)"
)
parser.add_argument(
"--ota",
nargs="+",
metavar=("HOST", "IMAGE"),
help="Update a v2 controller over Ethernet: HOST [IMAGE]. No serial port is used and the tool exits afterwards"
)
parser.add_argument(
"--flash",
metavar="BINARY",
@@ -538,6 +531,9 @@ Examples:
)
try:
if len(sys.argv) <= 1:
parser.print_help()
return
args = parser.parse_args()
ordered_actions = _get_ordered_actions(sys.argv)
except ValueError as e:
@@ -572,15 +568,6 @@ Examples:
print("Error: esptool not found. Install it with: pip install esptool", file=sys.stderr)
sys.exit(1)
elif action_name == 'ota':
try:
ota_update(value[0], value[1] if len(value) > 1 else None)
except Exception as e:
print(f"Error: OTA failed: {e}", file=sys.stderr)
sys.exit(1)
if all(a[0] in ('ota', 'pause') for a in ordered_actions):
return # network-only: nothing to download over serial
elif action_name == 'reset':
ran_reset = True
try:
@@ -610,8 +597,14 @@ Examples:
else:
print(f"Uploading {upload_dir} to device on {port}...", file=sys.stderr)
conn = DeviceConnection(port)
files_copied, dirs_created = conn.upload_directory(upload_dir, remote_dir)
print(f"Upload complete: {files_copied} files, {dirs_created} directories created.", file=sys.stderr)
files_copied, dirs_created, files_skipped = conn.upload_directory(
upload_dir, remote_dir, force=args.force_upload
)
print(
f"Upload complete: {files_copied} uploaded, {files_skipped} skipped, "
f"{dirs_created} directories created.",
file=sys.stderr,
)
except Exception as e:
print(f"Error uploading directory: {e}", file=sys.stderr)
sys.exit(1)
@@ -639,9 +632,12 @@ Examples:
file=sys.stderr,
)
conn = DeviceConnection(port)
files_copied, dirs_created = conn.upload_directory(temp_src, "")
files_copied, dirs_created, files_skipped = conn.upload_directory(
temp_src, "", force=args.force_upload
)
print(
f"Upload complete: {files_copied} files, {dirs_created} directories created.",
f"Upload complete: {files_copied} uploaded, {files_skipped} skipped, "
f"{dirs_created} directories created.",
file=sys.stderr,
)
except Exception as e:
@@ -662,14 +658,12 @@ Examples:
elif action_name == 'erase_all':
try:
print(f"Erasing all code on device {port}...", file=sys.stderr)
print(f"Erasing device filesystem on {port}...", file=sys.stderr)
conn = DeviceConnection(port)
items = conn.list_files('')
for name, is_dir, size in items:
if name == "settings.json":
continue
conn.delete(name)
print("Erase complete.", file=sys.stderr)
print("Erase complete (device root is empty).", file=sys.stderr)
except Exception as e:
print(f"Error erasing device: {e}", file=sys.stderr)
sys.exit(1)
@@ -766,32 +760,86 @@ Examples:
# Clamp into single-byte range; store as int in settings.json
edits["id"] = max(0, min(255, args.device_id))
# 1. Download: get current settings from device
try:
print(f"Downloading settings from {args.port}...", file=sys.stderr)
settings = download_settings(args.port)
print("Settings downloaded successfully.", file=sys.stderr)
except Exception as e:
if "timeout" in str(e).lower() or "connection" in str(e).lower():
print(f"Error: Connection timeout. Check device connection on {args.port}", file=sys.stderr)
else:
print(f"Error downloading settings: {e}", file=sys.stderr)
sys.exit(1)
settings_work = args.show or bool(edits)
if settings_work:
try:
print(f"Downloading settings from {args.port}...", file=sys.stderr)
settings = download_settings(args.port)
print("Settings downloaded successfully.", file=sys.stderr)
except Exception as e:
if "timeout" in str(e).lower() or "connection" in str(e).lower():
print(
f"Error: Connection timeout. Check device connection on {args.port}",
file=sys.stderr,
)
else:
print(f"Error downloading settings: {e}", file=sys.stderr)
sys.exit(1)
if args.show:
print_settings(settings)
if not edits and args.preset is None:
return
changed_edits: Dict[str, Any] = {}
for key, value in edits.items():
if settings.get(key) != value:
changed_edits[key] = value
if edits and not changed_edits:
print("No settings changes detected; skipping settings upload.", file=sys.stderr)
if changed_edits:
print(f"Applying {len(changed_edits)} setting change(s)...", file=sys.stderr)
settings.update(changed_edits)
# If --show only, print and exit (unless presets or settings are being written)
if args.show:
print_settings(settings)
if not edits and args.preset is None:
return
# 2. Edit: apply edits to downloaded settings
if edits:
print(f"Applying {len(edits)} edit(s)...", file=sys.stderr)
settings.update(edits)
if args.preset is not None:
pattern = args.pattern if args.pattern is not None else "on"
try:
print(f"Downloading presets from {args.port}...", file=sys.stderr)
presets_data = download_presets(args.port)
entry = presets_data.get(args.preset)
if not isinstance(entry, dict):
entry = {}
entry["p"] = pattern
presets_data[args.preset] = entry
print(
f"Writing preset {args.preset!r} (pattern={pattern}) to {PRESETS_REMOTE}...",
file=sys.stderr,
)
upload_presets(args.port, presets_data, reset=not bool(edits))
print("Presets uploaded successfully.", file=sys.stderr)
if not edits:
print("Device will reset.", file=sys.stderr)
except Exception as e:
if "timeout" in str(e).lower() or "connection" in str(e).lower():
print(
f"Error: Connection timeout. Check device connection on {args.port}",
file=sys.stderr,
)
else:
print(f"Error uploading presets: {e}", file=sys.stderr)
sys.exit(1)
print_settings(settings)
if changed_edits:
try:
print(f"\nUploading settings to {args.port}...", file=sys.stderr)
upload_settings(args.port, settings)
print("Settings uploaded successfully. Device will reset.", file=sys.stderr)
except Exception as e:
if "timeout" in str(e).lower() or "connection" in str(e).lower():
print(
f"Error: Connection timeout. Check device connection on {args.port}",
file=sys.stderr,
)
else:
print(f"Error uploading settings: {e}", file=sys.stderr)
sys.exit(1)
return
# 3a. Presets file (led-driver presets.json)
if args.preset is not None:
pattern = args.pattern if args.pattern is not None else "on"
try:
@@ -806,29 +854,24 @@ Examples:
f"Writing preset {args.preset!r} (pattern={pattern}) to {PRESETS_REMOTE}...",
file=sys.stderr,
)
upload_presets(args.port, presets_data, reset=not bool(edits))
upload_presets(args.port, presets_data, reset=True)
print("Presets uploaded successfully.", file=sys.stderr)
if not edits:
print("Device will reset.", file=sys.stderr)
print("Device will reset.", file=sys.stderr)
except Exception as e:
if "timeout" in str(e).lower() or "connection" in str(e).lower():
print(f"Error: Connection timeout. Check device connection on {args.port}", file=sys.stderr)
print(
f"Error: Connection timeout. Check device connection on {args.port}",
file=sys.stderr,
)
else:
print(f"Error uploading presets: {e}", file=sys.stderr)
sys.exit(1)
return
# 3b. Settings upload (resets device)
if edits:
try:
print(f"\nUploading settings to {args.port}...", file=sys.stderr)
upload_settings(args.port, settings)
print("Settings uploaded successfully. Device will reset.", file=sys.stderr)
except Exception as e:
if "timeout" in str(e).lower() or "connection" in str(e).lower():
print(f"Error: Connection timeout. Check device connection on {args.port}", file=sys.stderr)
else:
print(f"Error uploading settings: {e}", file=sys.stderr)
sys.exit(1)
if ordered_actions:
return
parser.print_help()
if __name__ == "__main__":
+45
View File
@@ -0,0 +1,45 @@
"""
Host-side helpers for file_hashes.json (same format as led-driver/src/file_hashes.py).
"""
import hashlib
import json
import os
MANIFEST_VERSION = 1
MANIFEST_FILENAME = "file_hashes.json"
HASH_ALGO = "sha256"
def normalize_remote_path(remote_file: str) -> str:
"""Device-relative path with forward slashes (no leading slash)."""
return remote_file.replace("\\", "/").lstrip("/")
def sha256_hex_file(path: str) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(65536), b""):
h.update(chunk)
return h.hexdigest()
def parse_manifest(data: bytes) -> dict:
"""Return path -> hash map from manifest bytes."""
try:
doc = json.loads(data.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError):
return {}
if not isinstance(doc, dict):
return {}
files = doc.get("files")
return dict(files) if isinstance(files, dict) else {}
def build_manifest_bytes(files: dict) -> bytes:
doc = {
"version": MANIFEST_VERSION,
"algorithm": HASH_ALGO,
"files": files,
}
return json.dumps(doc, separators=(",", ":")).encode("utf-8")
+59 -5
View File
@@ -10,6 +10,14 @@ import os
import time
import serial
from deploy_manifest import (
MANIFEST_FILENAME,
build_manifest_bytes,
normalize_remote_path,
parse_manifest,
sha256_hex_file,
)
# Add lib directory to path - handle both normal execution and PyInstaller bundle
if getattr(sys, 'frozen', False):
# Running as a PyInstaller bundle
@@ -108,14 +116,42 @@ class DeviceConnection:
self._fs_writefile_with_wdt(remote_path, data)
finally:
self.disconnect()
def _manifest_remote_path(self) -> str:
return "/" + MANIFEST_FILENAME
def read_hash_manifest(self) -> dict:
"""Load path -> sha256 hex map from file_hashes.json on the device."""
remote = self._manifest_remote_path()
if not self.transport.fs_exists(remote):
return {}
try:
data = self.transport.fs_readfile(remote)
return parse_manifest(data)
except Exception:
return {}
def write_hash_manifest(self, files: dict) -> None:
"""Write merged file_hashes.json to the device root."""
self._fs_writefile_with_wdt(
self._manifest_remote_path(),
build_manifest_bytes(files),
)
def upload_directory(self, local_dir, remote_dir=None):
def upload_directory(self, local_dir, remote_dir=None, *, force: bool = False):
"""
Upload a directory recursively to the device.
Skips files whose sha256 matches file_hashes.json on the device unless
force is True. Updates the manifest after upload.
Args:
local_dir: Local directory path to upload
remote_dir: Remote directory path (default: root, uses basename of local_dir)
force: Upload every file even when the manifest hash matches
Returns:
(files_copied, dirs_created, files_skipped)
"""
import os
@@ -129,15 +165,19 @@ class DeviceConnection:
remote_dir = os.path.basename(os.path.abspath(local_dir))
# Ensure remote directory exists
if not self.transport.fs_exists(remote_dir):
if remote_dir and not self.transport.fs_exists(remote_dir):
self.transport.fs_mkdir(remote_dir)
manifest = {} if force else self.read_hash_manifest()
# Walk through local directory and copy files
files_copied = 0
files_skipped = 0
dirs_created = 0
for root, dirs, files in os.walk(local_dir):
# Calculate relative path from local_dir
# Never upload Python bytecode trees (MicroPython does not use them).
dirs[:] = [d for d in dirs if d != "__pycache__"]
rel_path = os.path.relpath(root, local_dir)
# Build remote path
@@ -156,8 +196,12 @@ class DeviceConnection:
self.transport.fs_mkdir(remote_base)
dirs_created += 1
# Copy files
# Copy files (skip bytecode; __pycache__ dirs are pruned above)
for file in files:
if file.endswith((".pyc", ".pyo")):
continue
if file == MANIFEST_FILENAME:
continue
local_file = os.path.join(root, file)
# Handle root directory case properly
if remote_base == '/':
@@ -165,13 +209,23 @@ class DeviceConnection:
else:
remote_file = '/'.join([remote_base, file])
manifest_key = normalize_remote_path(remote_file)
local_hash = sha256_hex_file(local_file)
if not force and manifest.get(manifest_key) == local_hash:
print(f"Skipping (unchanged): {remote_file}", file=sys.stderr)
files_skipped += 1
continue
print(f"Uploading: {remote_file}", file=sys.stderr)
with open(local_file, 'rb') as f:
data = f.read()
self._fs_writefile_with_wdt(remote_file, data)
manifest[manifest_key] = local_hash
files_copied += 1
return files_copied, dirs_created
self.write_hash_manifest(manifest)
return files_copied, dirs_created, files_skipped
finally:
self.disconnect()
-77
View File
@@ -1,77 +0,0 @@
"""
Over-the-air firmware update for the v2 ESP32-S3 controller (Ethernet, plain HTTP).
The controller takes the raw app image on POST /ota, writes it to the idle slot and reboots.
"""
import http.client
import json
import os
import time
from typing import Optional
IMAGE_NAME = "led_controller_v2.bin"
# Searched in order when no image is given (relative to where the tool is run, since the
# PyInstaller binary has no useful location of its own)
DEFAULT_IMAGES = [
os.path.join("v2", "build", IMAGE_NAME),
os.path.join("build", IMAGE_NAME),
os.path.join("..", "v2", "build", IMAGE_NAME),
]
def _default_image() -> Optional[str]:
return next((p for p in DEFAULT_IMAGES if os.path.isfile(p)), None)
def _info(host: str, port: int) -> Optional[dict]:
try:
conn = http.client.HTTPConnection(host, port, timeout=5)
conn.request("GET", "/ota")
r = conn.getresponse()
return json.loads(r.read().decode()) if r.status == 200 else None
except (OSError, ValueError):
return None
def ota_update(host: str, image: Optional[str] = None, port: int = 80) -> None:
"""Upload image to host and wait for it to come back. Raises RuntimeError on failure."""
image = image or _default_image()
if not image:
raise RuntimeError(f"no image given and {IMAGE_NAME} not found under ./v2/build, ./build or ../v2/build")
if not os.path.isfile(image):
raise RuntimeError(f"image not found: {image}")
size = os.path.getsize(image)
before = _info(host, port)
if before is None:
raise RuntimeError(f"no OTA endpoint at http://{host}:{port}/ota (is this the v2 controller?)")
print(f"Running {before['partition']} (built {before['built']})")
print(f"Uploading {image} ({size} bytes)...")
conn = http.client.HTTPConnection(host, port, timeout=60)
with open(image, "rb") as f:
conn.request("POST", "/ota", body=f, headers={
"Content-Length": str(size),
"Content-Type": "application/octet-stream",
})
r = conn.getresponse()
body = r.read().decode()
if r.status != 200:
try:
body = json.loads(body).get("error", body)
except ValueError:
pass
raise RuntimeError(f"update rejected ({r.status}): {body}")
print("Written, waiting for reboot...")
time.sleep(3)
for _ in range(30):
after = _info(host, port)
if after:
print(f"Back up on {after['partition']} (built {after['built']})")
if after["partition"] == before["partition"]:
raise RuntimeError("controller came back on the same slot: new image was rejected or rolled back")
return
time.sleep(1)
raise RuntimeError("controller did not come back within 30 s")
+52
View File
@@ -0,0 +1,52 @@
"""Tests for deploy_manifest helpers (host Python)."""
import json
import os
import tempfile
import unittest
from deploy_manifest import (
MANIFEST_FILENAME,
build_manifest_bytes,
normalize_remote_path,
parse_manifest,
sha256_hex_file,
)
class DeployManifestTests(unittest.TestCase):
def test_normalize_remote_path(self):
self.assertEqual(normalize_remote_path("/patterns/chase.py"), "patterns/chase.py")
self.assertEqual(normalize_remote_path("main.py"), "main.py")
def test_round_trip_manifest(self):
files = {"main.py": "abc", "patterns/x.py": "def"}
blob = build_manifest_bytes(files)
parsed = parse_manifest(blob)
self.assertEqual(parsed, files)
doc = json.loads(blob.decode())
self.assertEqual(doc["version"], 1)
self.assertEqual(doc["algorithm"], "sha256")
def test_parse_manifest_invalid(self):
self.assertEqual(parse_manifest(b"not json"), {})
self.assertEqual(parse_manifest(b'{"files": "nope"}'), {})
def test_sha256_hex_file(self):
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(b"hello")
path = f.name
try:
self.assertEqual(
sha256_hex_file(path),
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
)
finally:
os.unlink(path)
def test_manifest_filename(self):
self.assertEqual(MANIFEST_FILENAME, "file_hashes.json")
if __name__ == "__main__":
unittest.main()