feat(cli): skip unchanged files using device file_hashes.json

Read and update file_hashes.json on deploy; add --force-upload and tests.

Co-authored-by: Cursor <cursoragent@cursor.com>

README.md

@@ -25,13 +25,14 @@ Connection is always via **`-p` / `--port`** (default `/dev/ttyACM0`). There is
 | `-r`, `--reset` | Reset the device |
 | `-f`, `--follow` | Follow serial output (optional timeout seconds) |
 | `--pause` | Sleep N seconds (for chained actions) |
-| `-u`, `--upload` | Recursive upload: `-u SRC [DEST]` |
-| `--src`, `--lib` | Upload `src/` or a tree to `/lib` |
-| `-e` | Erase device code (keeps `settings.json`) |
+| `-u`, `--upload` | Recursive upload: `-u SRC [DEST]` (skips unchanged files via `file_hashes.json` on device) |
+| `--src`, `--lib`, `--all` | Deploy led-driver trees to flash root, `patterns/`, and `lib/` |
+| `--force-upload` | Upload every file; ignore `file_hashes.json` |
+| `-e`, `--erase` | Erase everything at device root (including `settings.json` and `presets.json`) |
 | `--rm` | Remove a path on the device |
 | `--flash` | Flash a firmware binary (uses **esptool** on the host) |
 
-**Default behaviour:** the tool always downloads `settings.json`, prints the merged view, and uploads again **only** when you pass setting edits (`--name`, `--leds`, …), **`--preset`**, or the relevant upload/flash/erase actions in order.
+**Settings I/O:** With no arguments, **`led-cli`** prints help (no device access). **`--show`** downloads `settings.json` and prints it (read-only). Setting fields (`--name`, `-b`, …) download once, merge, print, and **upload only when a value actually changed**. **`--preset`** alone only touches **`presets.json`** (no `settings.json` download). Ordered actions (`--reset`, `--upload`, `-e`, …) run without touching settings unless you also pass **`--show`** or setting / preset edits as above.
 
 Run **`python cli.py -h`** for the full epilog and argument list.
 

cli.py

@@ -2,8 +2,10 @@
 """
 LED Bar Configuration CLI Tool
 
-Command-line interface for downloading, editing, and uploading settings.json
-to/from MicroPython devices via mpremote.
+Command-line interface for editing settings.json on MicroPython devices via mpremote.
+Settings are downloaded from the device only when displaying (--show) or when applying
+setting changes; uploads occur only when something changed. Use --erase to clear the
+device filesystem (including settings).
 """
 
 import json
@@ -221,6 +223,10 @@ def _get_ordered_actions(argv: List[str]) -> List[tuple]:
             actions.append(('upload', ["lib", "lib"]))
             i += 1
             continue
+        if arg == '--force-upload':
+            # Handled via argparse; skip during action scan
+            i += 1
+            continue
         if arg == '--lib':
             # Upload local DIR (default: ./lib) to /lib on device
             local_dir = "lib"
@@ -279,10 +285,13 @@ def main() -> None:
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog="""
 Examples:
-  # Download and print current settings
+  # Show help (no device I/O)
   %(prog)s
 
-  # Download, edit device name and brightness, then upload
+  # Show current settings.json from device (read-only)
+  %(prog)s --show
+
+  # Edit device name and brightness, then upload only if values changed
   %(prog)s -n "LED-Strip-1" -b 128
 
   # Set multiple parameters
@@ -481,6 +490,12 @@ Examples:
         help="Upload ./src (excluding patterns), ./src/patterns, and ./lib."
     )
 
+    parser.add_argument(
+        "--force-upload",
+        action="store_true",
+        help="Upload every file; ignore file_hashes.json on the device",
+    )
+
     parser.add_argument(
         "--patterns", "--paterns",
         dest="patterns_dir",
@@ -500,7 +515,7 @@ Examples:
         "-e", "--erase",
         dest="erase_all",
         action="store_true",
-        help="Erase all code on the device (delete all files except settings.json)"
+        help="Erase the device filesystem: delete all files and directories at device root (including settings.json and presets.json)",
     )
 
     parser.add_argument(
@@ -516,6 +531,9 @@ Examples:
     )
 
     try:
+        if len(sys.argv) <= 1:
+            parser.print_help()
+            return
         args = parser.parse_args()
         ordered_actions = _get_ordered_actions(sys.argv)
     except ValueError as e:
@@ -579,8 +597,14 @@ Examples:
                 else:
                     print(f"Uploading {upload_dir} to device on {port}...", file=sys.stderr)
                 conn = DeviceConnection(port)
-                files_copied, dirs_created = conn.upload_directory(upload_dir, remote_dir)
-                print(f"Upload complete: {files_copied} files, {dirs_created} directories created.", file=sys.stderr)
+                files_copied, dirs_created, files_skipped = conn.upload_directory(
+                    upload_dir, remote_dir, force=args.force_upload
+                )
+                print(
+                    f"Upload complete: {files_copied} uploaded, {files_skipped} skipped, "
+                    f"{dirs_created} directories created.",
+                    file=sys.stderr,
+                )
             except Exception as e:
                 print(f"Error uploading directory: {e}", file=sys.stderr)
                 sys.exit(1)
@@ -608,9 +632,12 @@ Examples:
                         file=sys.stderr,
                     )
                     conn = DeviceConnection(port)
-                    files_copied, dirs_created = conn.upload_directory(temp_src, "")
+                    files_copied, dirs_created, files_skipped = conn.upload_directory(
+                        temp_src, "", force=args.force_upload
+                    )
                     print(
-                        f"Upload complete: {files_copied} files, {dirs_created} directories created.",
+                        f"Upload complete: {files_copied} uploaded, {files_skipped} skipped, "
+                        f"{dirs_created} directories created.",
                         file=sys.stderr,
                     )
             except Exception as e:
@@ -631,14 +658,12 @@ Examples:
 
         elif action_name == 'erase_all':
             try:
-                print(f"Erasing all code on device {port}...", file=sys.stderr)
+                print(f"Erasing device filesystem on {port}...", file=sys.stderr)
                 conn = DeviceConnection(port)
                 items = conn.list_files('')
                 for name, is_dir, size in items:
-                    if name == "settings.json":
-                        continue
                     conn.delete(name)
-                print("Erase complete.", file=sys.stderr)
+                print("Erase complete (device root is empty).", file=sys.stderr)
             except Exception as e:
                 print(f"Error erasing device: {e}", file=sys.stderr)
                 sys.exit(1)
@@ -735,40 +760,86 @@ Examples:
         # Clamp into single-byte range; store as int in settings.json
         edits["id"] = max(0, min(255, args.device_id))
 
-    # 1. Download: get current settings from device
-    try:
-        print(f"Downloading settings from {args.port}...", file=sys.stderr)
-        settings = download_settings(args.port)
-        print("Settings downloaded successfully.", file=sys.stderr)
-    except Exception as e:
-        if "timeout" in str(e).lower() or "connection" in str(e).lower():
-            print(f"Error: Connection timeout. Check device connection on {args.port}", file=sys.stderr)
-        else:
-            print(f"Error downloading settings: {e}", file=sys.stderr)
-        sys.exit(1)
+    settings_work = args.show or bool(edits)
+
+    if settings_work:
+        try:
+            print(f"Downloading settings from {args.port}...", file=sys.stderr)
+            settings = download_settings(args.port)
+            print("Settings downloaded successfully.", file=sys.stderr)
+        except Exception as e:
+            if "timeout" in str(e).lower() or "connection" in str(e).lower():
+                print(
+                    f"Error: Connection timeout. Check device connection on {args.port}",
+                    file=sys.stderr,
+                )
+            else:
+                print(f"Error downloading settings: {e}", file=sys.stderr)
+            sys.exit(1)
+
+        if args.show:
+            print_settings(settings)
+            if not edits and args.preset is None:
+                return
+
+        changed_edits: Dict[str, Any] = {}
+        for key, value in edits.items():
+            if settings.get(key) != value:
+                changed_edits[key] = value
+
+        if edits and not changed_edits:
+            print("No settings changes detected; skipping settings upload.", file=sys.stderr)
+
+        if changed_edits:
+            print(f"Applying {len(changed_edits)} setting change(s)...", file=sys.stderr)
+            settings.update(changed_edits)
 
-    # If --show only, print and exit (unless presets or settings are being written)
-    if args.show:
         print_settings(settings)
-        if not edits and args.preset is None:
-            return
 
-    # 2. Edit: only apply/upload settings when values actually change
-    changed_edits: Dict[str, Any] = {}
-    for key, value in edits.items():
-        if settings.get(key) != value:
-            changed_edits[key] = value
+        if args.preset is not None:
+            pattern = args.pattern if args.pattern is not None else "on"
+            try:
+                print(f"Downloading presets from {args.port}...", file=sys.stderr)
+                presets_data = download_presets(args.port)
+                entry = presets_data.get(args.preset)
+                if not isinstance(entry, dict):
+                    entry = {}
+                entry["p"] = pattern
+                presets_data[args.preset] = entry
+                print(
+                    f"Writing preset {args.preset!r} (pattern={pattern}) to {PRESETS_REMOTE}...",
+                    file=sys.stderr,
+                )
+                upload_presets(args.port, presets_data, reset=not bool(edits))
+                print("Presets uploaded successfully.", file=sys.stderr)
+                if not edits:
+                    print("Device will reset.", file=sys.stderr)
+            except Exception as e:
+                if "timeout" in str(e).lower() or "connection" in str(e).lower():
+                    print(
+                        f"Error: Connection timeout. Check device connection on {args.port}",
+                        file=sys.stderr,
+                    )
+                else:
+                    print(f"Error uploading presets: {e}", file=sys.stderr)
+                sys.exit(1)
 
-    if edits and not changed_edits:
-        print("No settings changes detected; skipping settings upload.", file=sys.stderr)
+        if changed_edits:
+            try:
+                print(f"\nUploading settings to {args.port}...", file=sys.stderr)
+                upload_settings(args.port, settings)
+                print("Settings uploaded successfully. Device will reset.", file=sys.stderr)
+            except Exception as e:
+                if "timeout" in str(e).lower() or "connection" in str(e).lower():
+                    print(
+                        f"Error: Connection timeout. Check device connection on {args.port}",
+                        file=sys.stderr,
+                    )
+                else:
+                    print(f"Error uploading settings: {e}", file=sys.stderr)
+                sys.exit(1)
+        return
 
-    if changed_edits:
-        print(f"Applying {len(changed_edits)} setting change(s)...", file=sys.stderr)
-        settings.update(changed_edits)
-
-    print_settings(settings)
-
-    # 3a. Presets file (led-driver presets.json)
     if args.preset is not None:
         pattern = args.pattern if args.pattern is not None else "on"
         try:
@@ -783,29 +854,24 @@ Examples:
                 f"Writing preset {args.preset!r} (pattern={pattern}) to {PRESETS_REMOTE}...",
                 file=sys.stderr,
             )
-            upload_presets(args.port, presets_data, reset=not bool(edits))
+            upload_presets(args.port, presets_data, reset=True)
             print("Presets uploaded successfully.", file=sys.stderr)
-            if not edits:
-                print("Device will reset.", file=sys.stderr)
+            print("Device will reset.", file=sys.stderr)
         except Exception as e:
             if "timeout" in str(e).lower() or "connection" in str(e).lower():
-                print(f"Error: Connection timeout. Check device connection on {args.port}", file=sys.stderr)
+                print(
+                    f"Error: Connection timeout. Check device connection on {args.port}",
+                    file=sys.stderr,
+                )
             else:
                 print(f"Error uploading presets: {e}", file=sys.stderr)
             sys.exit(1)
+        return
 
-    # 3b. Settings upload (resets device)
-    if changed_edits:
-        try:
-            print(f"\nUploading settings to {args.port}...", file=sys.stderr)
-            upload_settings(args.port, settings)
-            print("Settings uploaded successfully. Device will reset.", file=sys.stderr)
-        except Exception as e:
-            if "timeout" in str(e).lower() or "connection" in str(e).lower():
-                print(f"Error: Connection timeout. Check device connection on {args.port}", file=sys.stderr)
-            else:
-                print(f"Error uploading settings: {e}", file=sys.stderr)
-            sys.exit(1)
+    if ordered_actions:
+        return
+
+    parser.print_help()
 
 
 if __name__ == "__main__":

deploy_manifest.py

@@ -0,0 +1,45 @@
+"""
+Host-side helpers for file_hashes.json (same format as led-driver/src/file_hashes.py).
+"""
+
+import hashlib
+import json
+import os
+
+MANIFEST_VERSION = 1
+MANIFEST_FILENAME = "file_hashes.json"
+HASH_ALGO = "sha256"
+
+
+def normalize_remote_path(remote_file: str) -> str:
+    """Device-relative path with forward slashes (no leading slash)."""
+    return remote_file.replace("\\", "/").lstrip("/")
+
+
+def sha256_hex_file(path: str) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(65536), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def parse_manifest(data: bytes) -> dict:
+    """Return path -> hash map from manifest bytes."""
+    try:
+        doc = json.loads(data.decode("utf-8"))
+    except (UnicodeDecodeError, json.JSONDecodeError, TypeError, ValueError):
+        return {}
+    if not isinstance(doc, dict):
+        return {}
+    files = doc.get("files")
+    return dict(files) if isinstance(files, dict) else {}
+
+
+def build_manifest_bytes(files: dict) -> bytes:
+    doc = {
+        "version": MANIFEST_VERSION,
+        "algorithm": HASH_ALGO,
+        "files": files,
+    }
+    return json.dumps(doc, separators=(",", ":")).encode("utf-8")

device.py

@@ -10,6 +10,14 @@ import os
 import time
 import serial
 
+from deploy_manifest import (
+    MANIFEST_FILENAME,
+    build_manifest_bytes,
+    normalize_remote_path,
+    parse_manifest,
+    sha256_hex_file,
+)
+
 # Add lib directory to path - handle both normal execution and PyInstaller bundle
 if getattr(sys, 'frozen', False):
     # Running as a PyInstaller bundle
@@ -109,13 +117,41 @@ class DeviceConnection:
         finally:
             self.disconnect()
 
-    def upload_directory(self, local_dir, remote_dir=None):
+    def _manifest_remote_path(self) -> str:
+        return "/" + MANIFEST_FILENAME
+
+    def read_hash_manifest(self) -> dict:
+        """Load path -> sha256 hex map from file_hashes.json on the device."""
+        remote = self._manifest_remote_path()
+        if not self.transport.fs_exists(remote):
+            return {}
+        try:
+            data = self.transport.fs_readfile(remote)
+            return parse_manifest(data)
+        except Exception:
+            return {}
+
+    def write_hash_manifest(self, files: dict) -> None:
+        """Write merged file_hashes.json to the device root."""
+        self._fs_writefile_with_wdt(
+            self._manifest_remote_path(),
+            build_manifest_bytes(files),
+        )
+    
+    def upload_directory(self, local_dir, remote_dir=None, *, force: bool = False):
         """
         Upload a directory recursively to the device.
         
+        Skips files whose sha256 matches file_hashes.json on the device unless
+        force is True. Updates the manifest after upload.
+        
         Args:
             local_dir: Local directory path to upload
             remote_dir: Remote directory path (default: root, uses basename of local_dir)
+            force: Upload every file even when the manifest hash matches
+        
+        Returns:
+            (files_copied, dirs_created, files_skipped)
         """
         import os
         
@@ -129,15 +165,19 @@ class DeviceConnection:
                 remote_dir = os.path.basename(os.path.abspath(local_dir))
             
             # Ensure remote directory exists
-            if not self.transport.fs_exists(remote_dir):
+            if remote_dir and not self.transport.fs_exists(remote_dir):
                 self.transport.fs_mkdir(remote_dir)
             
+            manifest = {} if force else self.read_hash_manifest()
+            
             # Walk through local directory and copy files
             files_copied = 0
+            files_skipped = 0
             dirs_created = 0
             
             for root, dirs, files in os.walk(local_dir):
-                # Calculate relative path from local_dir
+                # Never upload Python bytecode trees (MicroPython does not use them).
+                dirs[:] = [d for d in dirs if d != "__pycache__"]
                 rel_path = os.path.relpath(root, local_dir)
                 
                 # Build remote path
@@ -156,8 +196,12 @@ class DeviceConnection:
                     self.transport.fs_mkdir(remote_base)
                     dirs_created += 1
                 
-                # Copy files
+                # Copy files (skip bytecode; __pycache__ dirs are pruned above)
                 for file in files:
+                    if file.endswith((".pyc", ".pyo")):
+                        continue
+                    if file == MANIFEST_FILENAME:
+                        continue
                     local_file = os.path.join(root, file)
                     # Handle root directory case properly
                     if remote_base == '/':
@@ -165,13 +209,23 @@ class DeviceConnection:
                     else:
                         remote_file = '/'.join([remote_base, file])
                     
+                    manifest_key = normalize_remote_path(remote_file)
+                    local_hash = sha256_hex_file(local_file)
+                    if not force and manifest.get(manifest_key) == local_hash:
+                        print(f"Skipping (unchanged): {remote_file}", file=sys.stderr)
+                        files_skipped += 1
+                        continue
+                    
                     print(f"Uploading: {remote_file}", file=sys.stderr)
                     with open(local_file, 'rb') as f:
                         data = f.read()
                     self._fs_writefile_with_wdt(remote_file, data)
+                    manifest[manifest_key] = local_hash
                     files_copied += 1
             
-            return files_copied, dirs_created
+            self.write_hash_manifest(manifest)
+            
+            return files_copied, dirs_created, files_skipped
         finally:
             self.disconnect()
     

tests/test_deploy_manifest.py

@@ -0,0 +1,52 @@
+"""Tests for deploy_manifest helpers (host Python)."""
+
+import json
+import os
+import tempfile
+import unittest
+
+from deploy_manifest import (
+    MANIFEST_FILENAME,
+    build_manifest_bytes,
+    normalize_remote_path,
+    parse_manifest,
+    sha256_hex_file,
+)
+
+
+class DeployManifestTests(unittest.TestCase):
+    def test_normalize_remote_path(self):
+        self.assertEqual(normalize_remote_path("/patterns/chase.py"), "patterns/chase.py")
+        self.assertEqual(normalize_remote_path("main.py"), "main.py")
+
+    def test_round_trip_manifest(self):
+        files = {"main.py": "abc", "patterns/x.py": "def"}
+        blob = build_manifest_bytes(files)
+        parsed = parse_manifest(blob)
+        self.assertEqual(parsed, files)
+        doc = json.loads(blob.decode())
+        self.assertEqual(doc["version"], 1)
+        self.assertEqual(doc["algorithm"], "sha256")
+
+    def test_parse_manifest_invalid(self):
+        self.assertEqual(parse_manifest(b"not json"), {})
+        self.assertEqual(parse_manifest(b'{"files": "nope"}'), {})
+
+    def test_sha256_hex_file(self):
+        with tempfile.NamedTemporaryFile(delete=False) as f:
+            f.write(b"hello")
+            path = f.name
+        try:
+            self.assertEqual(
+                sha256_hex_file(path),
+                "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
+            )
+        finally:
+            os.unlink(path)
+
+    def test_manifest_filename(self):
+        self.assertEqual(MANIFEST_FILENAME, "file_hashes.json")
+
+
+if __name__ == "__main__":
+    unittest.main()