Compare commits
51
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8c783ab926 | ||
|
|
f0615dc1b7 | ||
|
|
02440be55d | ||
|
|
a1c8ce853b | ||
|
|
581b751904 | ||
|
|
698c1463c0 | ||
|
|
cf3902845a | ||
|
|
93e986b2cb | ||
|
|
6e2d5ff80c | ||
|
|
c4f8a51359 | ||
|
|
50ca46fee6 | ||
|
|
4d8aeb5df8 | ||
|
|
3f8f67823b | ||
|
|
2543673bd8 | ||
|
|
d664026ee5 | ||
|
|
38162d4b89 | ||
|
|
d4118072f6 | ||
|
|
f85021d40d | ||
|
|
9989dd3062 | ||
|
|
06953028ab | ||
|
|
2f0c26e9b8 | ||
|
|
4372ba4105 | ||
|
|
f47237696e | ||
|
|
190c8bc982 | ||
|
|
b6aa0635ae | ||
|
|
a560a1a72b | ||
|
|
24d3c1149d | ||
|
|
16a3d3aa26 | ||
|
|
011883a6a8 | ||
|
|
b2cd242390 | ||
|
|
bfeeb958df | ||
|
|
04b56c6ef2 | ||
|
|
19fea0b0a1 | ||
|
|
e7f238aebc | ||
|
|
57182188ba | ||
|
|
4b42391a46 | ||
|
|
9d628aa3c8 | ||
|
|
32c0ecfa76 | ||
|
|
1c222771a5 | ||
|
|
71bd320661 | ||
|
|
6c16998691 | ||
|
|
8d78af1246 | ||
|
|
899d4f2418 | ||
|
|
f7ac4ab290 | ||
|
|
d332d447aa | ||
|
|
a94d6a7de5 | ||
|
|
7336c7fb22 | ||
|
|
3a7723c586 | ||
|
|
4fc3345fc9 | ||
|
|
42c14361e8 | ||
|
|
7ce56b64df |
No files matched your search
@@ -0,0 +1,3 @@
|
||||
# graphify
|
||||
- **graphify** (`.claude/skills/graphify/SKILL.md`) - any input to knowledge graph. Trigger: `/graphify`
|
||||
When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else.
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"version": "0.0.1",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "led-controller",
|
||||
"url": "http://localhost:8080",
|
||||
"port": 8080
|
||||
},
|
||||
{
|
||||
"name": "led-controller-lan",
|
||||
"url": "http://10.1.1.223:8080",
|
||||
"port": 8080
|
||||
},
|
||||
{
|
||||
"name": "led-controller-sim",
|
||||
"runtimeExecutable": "bash",
|
||||
"runtimeArgs": [
|
||||
"-c",
|
||||
"mkdir -p sim-controller/data && LED_SIM_DB=$PWD/sim-controller/data LED_SIM_BRIDGE=browser LED_SIM_HTTP_PORT=8081 exec sim-controller/build/led_controller_sim.elf"
|
||||
],
|
||||
"port": 8081
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
0.9.101
|
||||
@@ -0,0 +1,797 @@
|
||||
---
|
||||
name: graphify
|
||||
description: "Use for any question about a codebase, its architecture, file relationships, or project content — especially when graphify-out/ exists, where the question should be treated as a graphify query first. Turns any input (code, docs, papers, images, videos) into a persistent knowledge graph with god nodes, community detection, and query/path/explain tools."
|
||||
---
|
||||
|
||||
# /graphify
|
||||
|
||||
Turn any folder of files into a navigable knowledge graph with community detection, an honest audit trail, and three outputs: interactive HTML, GraphRAG-ready JSON, and a plain-language GRAPH_REPORT.md.
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault)
|
||||
/graphify <path> # full pipeline on specific path
|
||||
/graphify https://github.com/<owner>/<repo> # clone repo then run full pipeline on it
|
||||
/graphify https://github.com/<owner>/<repo> --branch <branch> # clone a specific branch
|
||||
/graphify <url1> <url2> ... # clone multiple repos, build each, merge into one cross-repo graph
|
||||
/graphify <path> --mode deep # thorough extraction, richer INFERRED edges
|
||||
/graphify <path> --update # incremental - re-extract only new/changed files
|
||||
/graphify <path> --directed # build directed graph (preserves edge direction: source→target)
|
||||
/graphify <path> --whisper-model medium # use a larger Whisper model for better transcription accuracy
|
||||
/graphify <path> --cluster-only # rerun clustering on existing graph
|
||||
/graphify <path> --no-viz # skip visualization, just report + JSON
|
||||
/graphify <path> --html # (HTML is generated by default - this flag is a no-op)
|
||||
/graphify <path> --svg # also export graph.svg (embeds in Notion, GitHub)
|
||||
/graphify <path> --graphml # export graph.graphml (Gephi, yEd)
|
||||
/graphify <path> --neo4j # generate graphify-out/cypher.txt for Neo4j
|
||||
/graphify <path> --neo4j-push bolt://localhost:7687 # push directly to Neo4j
|
||||
/graphify <path> --falkordb # generate graphify-out/cypher.txt for FalkorDB
|
||||
/graphify <path> --falkordb-push falkordb://localhost:6379 # push directly to FalkorDB
|
||||
/graphify <path> --mcp # start MCP stdio server for agent access
|
||||
/graphify <path> --watch # watch folder, auto-rebuild on code changes (no LLM needed)
|
||||
/graphify <path> --wiki # build agent-crawlable wiki (index.md + one article per community)
|
||||
/graphify <path> --obsidian --obsidian-dir ~/vaults/my-project # write vault to custom path (e.g. existing vault)
|
||||
/graphify add <url> # fetch URL, save to ./raw, update graph
|
||||
/graphify add <url> --author "Name" # tag who wrote it
|
||||
/graphify add <url> --contributor "Name" # tag who added it to the corpus
|
||||
/graphify query "<question>" # BFS traversal - broad context
|
||||
/graphify query "<question>" --dfs # DFS - trace a specific path
|
||||
/graphify query "<question>" --budget 1500 # cap answer at N tokens
|
||||
/graphify path "AuthModule" "Database" # shortest path between two concepts
|
||||
/graphify explain "SwinTransformer" # plain-language explanation of a node
|
||||
```
|
||||
|
||||
## What graphify is for
|
||||
|
||||
Drop any folder of code, docs, papers, images, or video into graphify and get a queryable knowledge graph. Persistent across sessions, honest audit trail (EXTRACTED/INFERRED/AMBIGUOUS), community detection surfaces cross-document connections you wouldn't think to ask about.
|
||||
|
||||
**When a graph is worth building:** this applies when you are choosing a tool yourself; if the user invoked `/graphify`, follow that request as asked. AST extraction of code is fast and free, but semantic extraction of docs, papers and images runs the model over every file, which can cost minutes and tens of thousands of tokens.
|
||||
|
||||
If a graph already exists, query it. With no graph, one targeted lookup ("where is X defined", "what calls Y" in an ordinary-sized codebase) is faster and free with a plain text search such as `grep` and `Read`: search directly instead of running the steps below, and mention that a graph can be built if the user wants one. Build one when you expect many questions against this corpus, the corpus is large or unfamiliar, the question needs links text search cannot see (doc to code, across documents), or it is structural (communities, bridges, import cycles).
|
||||
|
||||
## What You Must Do When Invoked
|
||||
|
||||
If the user invoked `/graphify --help` or `/graphify -h` (with no other arguments), print the contents of the `## Usage` section above verbatim and stop. Do not run any commands, do not detect files, do not default the path to `.`. Just print the Usage block and return.
|
||||
|
||||
**Fast path — existing graph:** Before doing anything else, check whether `graphify-out/graph.json` exists. The expected location is `graphify-out/graph.json` relative to the **current working directory** (i.e. the project root where you are running commands). If it exists AND the user's request is a natural-language question about the codebase (e.g. "How does X work?", "What calls Y?", "Trace the data flow through Z") and NOT an explicit rebuild command (`--update`, `--cluster-only`, or a bare path/URL that implies fresh extraction): **skip Steps 1–5 entirely and jump straight to `## For /graphify query`.** Run `graphify query "<question>"` immediately. Do not run detect. Do not check corpus size. Do not ask the user to narrow. The graph is already built — use it.
|
||||
|
||||
If no path was given, use `.` (current directory). Do not ask the user for a path.
|
||||
|
||||
If the path argument starts with `https://github.com/` or `http://github.com/`, treat it as a GitHub URL - run Step 0 before anything else, then continue with the resolved local path.
|
||||
|
||||
Follow these steps in order unless the fast path above applies.
|
||||
|
||||
### Step 0 - GitHub repos and multi-path merge (only if a URL or several paths)
|
||||
|
||||
Only when the path is one or more `https://github.com/...` URLs, or several local subfolders to merge. See `references/github-and-merge.md` for the clone, cross-repo merge, and monorepo flow, then continue with the resolved local path. A plain local path skips this step.
|
||||
|
||||
### Step 1 - Ensure graphify is installed
|
||||
|
||||
```bash
|
||||
# Detect the correct Python interpreter (handles uv tool, pipx, venv, system installs)
|
||||
PYTHON=""
|
||||
GRAPHIFY_BIN=$(which graphify 2>/dev/null)
|
||||
# 1. uv tool installs — most reliable on modern Mac/Linux
|
||||
if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then
|
||||
_UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null)
|
||||
if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi
|
||||
fi
|
||||
# 2. Read shebang from graphify binary (pipx and direct pip installs)
|
||||
if [ -z "$PYTHON" ] && [ -n "$GRAPHIFY_BIN" ]; then
|
||||
_SHEBANG=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!')
|
||||
case "$_SHEBANG" in
|
||||
*[!a-zA-Z0-9/_.@-]*) ;;
|
||||
*) "$_SHEBANG" -c "import graphify" 2>/dev/null && PYTHON="$_SHEBANG" ;;
|
||||
esac
|
||||
fi
|
||||
# 3. Fall back to python3
|
||||
if [ -z "$PYTHON" ]; then PYTHON="python3"; fi
|
||||
if ! "$PYTHON" -c "import graphify" 2>/dev/null; then
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv tool install --upgrade graphifyy -q 2>&1 | tail -3
|
||||
_UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null)
|
||||
if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi
|
||||
else
|
||||
"$PYTHON" -m pip install graphifyy -q 2>/dev/null \
|
||||
|| "$PYTHON" -m pip install graphifyy -q --break-system-packages 2>&1 | tail -3
|
||||
fi
|
||||
fi
|
||||
# Write interpreter path for all subsequent steps (persists across invocations)
|
||||
mkdir -p graphify-out
|
||||
"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)"
|
||||
# Save scan root so `graphify update` (no args) knows where to look next time.
|
||||
# The scan path is passed through a quoted heredoc, never substituted into the
|
||||
# command line itself: a bare `cd <path>` (or an unquoted heredoc, which
|
||||
# still expands $()/backticks in its body) would let a malicious path execute
|
||||
# as shell code the moment this line runs.
|
||||
"$PYTHON" -c "import os, sys; out_path = os.path.abspath('graphify-out/.graphify_root'); os.chdir(sys.stdin.readline().rstrip('\n')); open(out_path, 'w', encoding='utf-8').write(os.getcwd())" <<'GRAPHIFY_ROOT_EOF'
|
||||
INPUT_PATH
|
||||
GRAPHIFY_ROOT_EOF
|
||||
```
|
||||
|
||||
If the import succeeds, print nothing and move straight to Step 2.
|
||||
|
||||
**In every subsequent bash block, replace `python3` with `$(cat graphify-out/.graphify_python)` to use the correct interpreter.**
|
||||
|
||||
### Step 2 - Detect files
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from graphify.detect import detect
|
||||
from pathlib import Path
|
||||
from graphify.paths import out_path
|
||||
from graphify.watch import _read_build_excludes, _read_build_gitignore
|
||||
result = detect(Path('INPUT_PATH'), extra_excludes=_read_build_excludes(out_path()), gitignore=_read_build_gitignore(out_path()))
|
||||
# Write the sidecar from Python, not a shell redirect, so the same block renders
|
||||
# on PowerShell hosts without console-encoding drift (#2528).
|
||||
Path('graphify-out/.graphify_detect.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Detected {result[\"total_files\"]} files')
|
||||
"
|
||||
```
|
||||
|
||||
Replace INPUT_PATH with the actual path the user provided. Do NOT cat or print the JSON - read it silently and present a clean summary instead:
|
||||
|
||||
```
|
||||
Corpus: X files · ~Y words
|
||||
code: N files (.py .ts .go ...)
|
||||
docs: N files (.md .txt ...)
|
||||
papers: N files (.pdf ...)
|
||||
images: N files
|
||||
video: N files (.mp4 .mp3 ...)
|
||||
```
|
||||
|
||||
Omit any category with 0 files from the summary.
|
||||
|
||||
Then act on it:
|
||||
- If `total_files` is 0: stop with "No supported files found in [path]."
|
||||
- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106).
|
||||
- If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count:
|
||||
- Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH).
|
||||
- Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`).
|
||||
- Filter out any path that starts with `scan_root + "/graphify-out/"` to exclude converted sidecars.
|
||||
- For each file, strip the `scan_root` prefix and take the first path component. Files directly in `scan_root` with no subdirectory count as `(root)`.
|
||||
- If all files are in `(root)` with no subdirectories, do not ask to narrow — no subfolders exist. Instead suggest `--no-cluster` to skip the expensive clustering step and proceed.
|
||||
- Otherwise rank by count, show the top 5 with file counts, then ask which subfolder to run on. Wait for the user's answer before proceeding.
|
||||
- Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not.
|
||||
|
||||
### Step 2.5 - Video and audio (only if video files detected)
|
||||
|
||||
Skip this step entirely if `detect` returned zero `video` files. When the corpus has video or audio, see `references/transcribe.md` to transcribe them to text first, then treat the transcripts as doc files in Step 3.
|
||||
|
||||
### Step 3 - Extract entities and relationships
|
||||
|
||||
**Before starting:** note whether `--mode deep` was given. You must pass `DEEP_MODE=true` to every subagent in Step B2 if it was. Track this from the original invocation - do not lose it.
|
||||
|
||||
This step has two parts: **structural extraction** (deterministic, free) and **semantic extraction** (LLM, costs tokens).
|
||||
|
||||
> **graphify needs no API key. Never ask the user for one, and never block on one.** Code is extracted structurally (AST) with no LLM and no key at all — a code-only corpus (the common `/graphify .` on a repo) skips semantic extraction entirely, so it needs nothing here: go straight to Part A and skip Part B. Semantic extraction (only for docs, papers, and images) uses Gemini **only if** `GEMINI_API_KEY`/`GOOGLE_API_KEY` is already set; otherwise the host agent itself is the LLM. graphify does **not** read `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, or any other provider key. If you catch yourself about to prompt for, wait on, or stop because of a missing API key, that is a misread of this skill — proceed without one.
|
||||
|
||||
**Before semantic extraction:** check whether `GEMINI_API_KEY` or `GOOGLE_API_KEY` is set. If neither is set, print this one-liner to the user:
|
||||
> Tip: set `GEMINI_API_KEY` or `GOOGLE_API_KEY` to use Gemini for semantic extraction (`pip install 'graphifyy[gemini]'`).
|
||||
|
||||
Print it once, then continue — do not wait for the user to supply a key. If `GEMINI_API_KEY` or `GOOGLE_API_KEY` IS set, use `graphify.llm.extract_corpus_parallel(files, backend="gemini")` for semantic extraction instead of dispatching subagents. The default Gemini model is `gemini-3-flash-preview`; set `GRAPHIFY_GEMINI_MODEL` or pass `--model` in headless CLI flows to override it.
|
||||
|
||||
> **No other API keys are read.** When `GEMINI_API_KEY`/`GOOGLE_API_KEY` are unset, semantic extraction falls to the host agent itself — the running session is the LLM. On a host that dispatches subagents (e.g. Claude Code), dispatch them as written in Part B. On a host that runs the CLI directly in a terminal and cannot dispatch subagents, do not stall: a code-only corpus has no semantic work, so write the empty semantic file (Part B "Fast path") and continue to Part C; for a corpus with docs/papers/images, either set a Gemini key or extract those inline yourself, but in no case prompt for `ANTHROPIC_API_KEY` — that prompt is a misread of this skill.
|
||||
|
||||
**Run Part A (AST) and Part B (semantic) in parallel. Dispatch all semantic subagents AND start AST extraction in the same message. Both can run simultaneously since they operate on different file types. Merge results in Part C as before.**
|
||||
|
||||
Note: Parallelizing AST + semantic saves 5-15s on large corpora. AST is deterministic and fast; start it while subagents are processing docs/papers.
|
||||
|
||||
#### Part A - Structural extraction for code files
|
||||
|
||||
For any code files detected, run AST extraction in parallel with Part B subagents:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from graphify.extract import collect_files, extract
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
code_files = []
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
for f in detect.get('files', {}).get('code', []):
|
||||
code_files.extend(collect_files(Path(f)) if Path(f).is_dir() else [Path(f)])
|
||||
|
||||
if code_files:
|
||||
# root= anchors node ids and source_file on the scan root; cache_root only says
|
||||
# where graphify-out/cache lives. Pass both: without root= the anchor falls back
|
||||
# to cache_root, so pointing the cache elsewhere would rename every node (#3727).
|
||||
result = extract(code_files, cache_root=Path('INPUT_PATH'), root=Path('INPUT_PATH'))
|
||||
Path('graphify-out/.graphify_ast.json').write_text(json.dumps(result, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'AST: {len(result[\"nodes\"])} nodes, {len(result[\"edges\"])} edges')
|
||||
else:
|
||||
Path('graphify-out/.graphify_ast.json').write_text(json.dumps({'nodes':[],'edges':[],'input_tokens':0,'output_tokens':0}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print('No code files - skipping AST extraction')
|
||||
"
|
||||
```
|
||||
|
||||
#### Part B - Semantic extraction (parallel subagents)
|
||||
|
||||
**Fast path:** If detection found zero docs, papers, and images (code-only corpus), skip Part B entirely and go straight to Part C. AST handles code - there is nothing for semantic subagents to do. **First write an empty semantic file** so Part C's merge has its input (it reads `.graphify_semantic.json` unconditionally; without this a code-only run hits `FileNotFoundError`):
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
Path('graphify-out/.graphify_semantic.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8')
|
||||
"
|
||||
```
|
||||
|
||||
**MANDATORY: You MUST use the Agent tool here. Reading files yourself one-by-one is forbidden - it is 5-10x slower. If you do not use the Agent tool you are doing this wrong.**
|
||||
|
||||
Before dispatching subagents, print a timing estimate:
|
||||
- Load `total_words` and file counts from `graphify-out/.graphify_detect.json`
|
||||
- Estimate agents needed: `ceil(uncached_non_code_files / 22)` (chunk size is 20-25 files, capped at ~6000 total lines)
|
||||
- Estimate time: ~45s per agent batch (they run in parallel, so total ≈ 45s × ceil(agents/parallel_limit))
|
||||
- Print: "Semantic extraction: ~N files → X agents, estimated ~Ys"
|
||||
|
||||
**Step B0 - Check extraction cache first**
|
||||
|
||||
Before dispatching any subagents, check which files already have cached extraction results:
|
||||
|
||||
SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument.
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from graphify.cache import check_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
# Only content files go to semantic extraction. Code is already covered structurally
|
||||
# by the AST pass (Part A); flattening every category here makes subagents re-read
|
||||
# every source file (#1392). Video is transcribed to a document in Step 2.5 first.
|
||||
all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])]
|
||||
|
||||
cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH')
|
||||
|
||||
# Always (re)write the cache file: write hits, else DELETE any leftover from a prior
|
||||
# run so Part C never merges a stale .graphify_cached.json (#1392).
|
||||
if cached_nodes or cached_edges or cached_hyperedges:
|
||||
Path('graphify-out/.graphify_cached.json').write_text(json.dumps({'nodes': cached_nodes, 'edges': cached_edges, 'hyperedges': cached_hyperedges}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
else:
|
||||
Path('graphify-out/.graphify_cached.json').unlink(missing_ok=True)
|
||||
Path('graphify-out/.graphify_uncached.txt').write_text('\n'.join(uncached), encoding=\"utf-8\")
|
||||
# Nothing has been dispatched yet, so any chunk file on disk is a leftover from an
|
||||
# interrupted run, and Step B3 merges every .graphify_chunk_*.json it finds.
|
||||
for stale in Path('graphify-out').glob('.graphify_chunk_*.json'):
|
||||
stale.unlink()
|
||||
print(f'Cache: {len(all_files)-len(uncached)} files hit, {len(uncached)} files need extraction')
|
||||
"
|
||||
```
|
||||
|
||||
Only dispatch subagents for files listed in `graphify-out/.graphify_uncached.txt`. If all files are cached, skip Steps B1 and B2 but still run Step B3's commands: its merge is the only Part B step that writes `graphify-out/.graphify_semantic.json`, and Part C reads that file unconditionally. Do not write an empty `.graphify_semantic.json` instead, because that drops every cached node.
|
||||
|
||||
**Step B1 - Split into chunks**
|
||||
|
||||
Load files from `graphify-out/.graphify_uncached.txt`. Split into chunks bounded by BOTH limits: at most 20-25 files AND at most ~6000 total lines per chunk. A text file longer than 2000 lines gets a chunk to itself - that is past the Read tool's single-call cap, so sharing a chunk with other files guarantees it is only partly read. Each image gets its own chunk (vision needs separate context). When splitting, group files from the same directory together so related artifacts land in the same chunk and cross-file relationships are more likely to be extracted.
|
||||
|
||||
Both bounds matter. The line bound is not only about the 2000-line Read cap: a subagent holding 25 files splits its attention across all of them and skims the dense ones, and a 590-line plan can come back with zero nodes that way. Bounding content as well as file count is what keeps a chunk's coverage honest (#2225).
|
||||
|
||||
**Step B2 - Dispatch ALL subagents in a single message**
|
||||
|
||||
Call the Agent tool multiple times IN THE SAME RESPONSE - one call per chunk. This is the only way they run in parallel. If you make one Agent call, wait, then make another, you are doing it sequentially and defeating the purpose.
|
||||
|
||||
**IMPORTANT - subagent type:** Always use `subagent_type="general-purpose"`. Do NOT use `Explore` - it is read-only and cannot write chunk files to disk, which silently drops extraction results. General-purpose has Write and Bash access which the subagent needs.
|
||||
|
||||
Concrete example for 3 chunks:
|
||||
```
|
||||
[Agent tool call 1: files 1-15, subagent_type="general-purpose"]
|
||||
[Agent tool call 2: files 16-30, subagent_type="general-purpose"]
|
||||
[Agent tool call 3: files 31-45, subagent_type="general-purpose"]
|
||||
```
|
||||
All three in one message. Not three separate messages.
|
||||
|
||||
Each subagent receives this exact prompt (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH).
|
||||
|
||||
CHUNK_PATH must be an **absolute** path — derive it before dispatching:
|
||||
```bash
|
||||
PROJECT_ROOT=$(pwd) # cwd — where Part C globs graphify-out/ (NOT .graphify_root/scan dir, #1392)
|
||||
# Then for chunk N: CHUNK_PATH="${PROJECT_ROOT}/graphify-out/.graphify_chunk_0N.json"
|
||||
```
|
||||
|
||||
Subagent prompt template:
|
||||
|
||||
See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH.
|
||||
|
||||
**Step B3 - Collect, cache, and merge**
|
||||
|
||||
Wait for all subagents. For each result:
|
||||
- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal
|
||||
- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache
|
||||
- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip.
|
||||
- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort
|
||||
|
||||
If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used.
|
||||
|
||||
**Step B3.5 - Verify each reported file was actually read to its end**
|
||||
|
||||
A subagent that only read the first Read call's worth of a long file still writes a well-formed chunk, so "chunk file exists and parses" is not evidence of coverage. Compare each file's self-reported `coverage` against its real length:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, glob
|
||||
from pathlib import Path
|
||||
|
||||
covered = {}
|
||||
for c in sorted(glob.glob('graphify-out/.graphify_chunk_*.json')):
|
||||
d = json.loads(Path(c).read_text(encoding=\"utf-8\"))
|
||||
for src, n in (d.get('coverage') or {}).items():
|
||||
covered[src] = max(covered.get(src, 0), int(n))
|
||||
|
||||
partial = []
|
||||
for src, n in sorted(covered.items()):
|
||||
p = Path(src)
|
||||
if not p.is_absolute():
|
||||
p = Path('INPUT_PATH') / src
|
||||
if not p.is_file():
|
||||
continue
|
||||
total = sum(1 for _ in p.open(encoding='utf-8', errors='replace'))
|
||||
if total and n < total:
|
||||
partial.append((src, n, total))
|
||||
|
||||
Path('graphify-out/.graphify_partial.txt').write_text('\n'.join(s for s, _, _ in partial), encoding=\"utf-8\")
|
||||
if partial:
|
||||
print(f'WARNING: {len(partial)} file(s) were only partly read. They are cached as partial and retried on the next --update:')
|
||||
for src, n, total in partial:
|
||||
print(f' {src}: read {n} of {total} lines')
|
||||
else:
|
||||
print('Coverage OK - every file a subagent reported was read to its last line')
|
||||
"
|
||||
```
|
||||
|
||||
Tell the user which files came up short and what was lost (the line counts are printed above). The nodes already extracted for those files are kept - they are still true - but the file is cached as partial and left unstamped in the manifest below, so the next `--update` re-dispatches it and replace-on-re-extract swaps the partial node set for a complete one. This is the same mechanism the library path uses for a truncated LLM response (#933, #1950), so both paths now recover instead of silently keeping a half-read file forever. A file a subagent did not report at all is not counted as partial - there is nothing to compare against - which is why the prompt rule in `references/extraction-spec.md` asks for the count in the first place.
|
||||
|
||||
Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run:
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, glob
|
||||
from pathlib import Path
|
||||
|
||||
chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json'))
|
||||
all_nodes, all_edges, all_hyperedges = [], [], []
|
||||
total_in, total_out = 0, 0
|
||||
for c in chunks:
|
||||
d = json.loads(Path(c).read_text(encoding=\"utf-8\"))
|
||||
all_nodes += d.get('nodes', [])
|
||||
all_edges += d.get('edges', [])
|
||||
all_hyperedges += d.get('hyperedges', [])
|
||||
total_in += d.get('input_tokens', 0)
|
||||
total_out += d.get('output_tokens', 0)
|
||||
Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({
|
||||
'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges,
|
||||
'input_tokens': total_in, 'output_tokens': total_out,
|
||||
}, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens')
|
||||
"
|
||||
```
|
||||
|
||||
Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939):
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from graphify.cache import save_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
|
||||
# A partly-read file is cached with partial: True, which load_cached treats as a MISS,
|
||||
# so the next --update re-dispatches it instead of serving a truncated extraction forever
|
||||
# (#2225). Same contract the library path uses for a truncated LLM response (#933/#1950).
|
||||
partial = [line for line in Path('graphify-out/.graphify_partial.txt').read_text(encoding=\"utf-8\").splitlines() if line] if Path('graphify-out/.graphify_partial.txt').exists() else []
|
||||
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH', partial_source_files=partial)
|
||||
note = f' ({len(partial)} left partial for retry)' if partial else ''
|
||||
print(f'Cached {saved} files' + note)
|
||||
"
|
||||
```
|
||||
|
||||
Merge cached + new results into `graphify-out/.graphify_semantic.json`:
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
|
||||
all_nodes = cached['nodes'] + new.get('nodes', [])
|
||||
all_edges = cached['edges'] + new.get('edges', [])
|
||||
all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', [])
|
||||
seen = set()
|
||||
deduped = []
|
||||
for n in all_nodes:
|
||||
if n['id'] not in seen:
|
||||
seen.add(n['id'])
|
||||
deduped.append(n)
|
||||
|
||||
merged = {
|
||||
'nodes': deduped,
|
||||
'edges': all_edges,
|
||||
'hyperedges': all_hyperedges,
|
||||
'input_tokens': new.get('input_tokens', 0),
|
||||
'output_tokens': new.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)')
|
||||
"
|
||||
```
|
||||
Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json`
|
||||
|
||||
#### Part C - Merge AST + semantic into final extraction
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from pathlib import Path
|
||||
|
||||
ast = json.loads(Path('graphify-out/.graphify_ast.json').read_text(encoding=\"utf-8\"))
|
||||
sem = json.loads(Path('graphify-out/.graphify_semantic.json').read_text(encoding=\"utf-8\"))
|
||||
|
||||
# Stamp tier provenance on the semantic side. build._is_ast_tier reads
|
||||
# _origin when present and otherwise guesses from the SHAPE of
|
||||
# source_location ('L<line>' means AST). A subagent that writes 'L12' for a
|
||||
# doc section therefore made that node read as AST, and the next
|
||||
# graphify update deleted it along with the re-extracted code file it
|
||||
# seemed to belong to (#2843). extract() stamps its own items 'ast'; the
|
||||
# semantic side must say so explicitly.
|
||||
for n in sem['nodes']:
|
||||
n.setdefault('_origin', 'semantic')
|
||||
for e in sem['edges']:
|
||||
e.setdefault('_origin', 'semantic')
|
||||
|
||||
# Merge: AST nodes first, semantic nodes deduplicated by id
|
||||
seen = {n['id'] for n in ast['nodes']}
|
||||
merged_nodes = list(ast['nodes'])
|
||||
for n in sem['nodes']:
|
||||
if n['id'] not in seen:
|
||||
merged_nodes.append(n)
|
||||
seen.add(n['id'])
|
||||
|
||||
merged_edges = ast['edges'] + sem['edges']
|
||||
merged_hyperedges = sem.get('hyperedges', [])
|
||||
merged = {
|
||||
'nodes': merged_nodes,
|
||||
'edges': merged_edges,
|
||||
'hyperedges': merged_hyperedges,
|
||||
'input_tokens': sem.get('input_tokens', 0),
|
||||
'output_tokens': sem.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
total = len(merged_nodes)
|
||||
edges = len(merged_edges)
|
||||
print(f'Merged: {total} nodes, {edges} edges ({len(ast[\"nodes\"])} AST + {len(sem[\"nodes\"])} semantic)')
|
||||
"
|
||||
```
|
||||
|
||||
### Step 4 - Build graph, cluster, analyze, generate outputs
|
||||
|
||||
**Before starting:** the code blocks below pass `directed=IS_DIRECTED` to `build_from_json()`. Replace `IS_DIRECTED` with `True` if `--directed` was given (builds a `DiGraph` preserving edge direction source→target), otherwise `False` (the default undirected `Graph`). Substitute it the same way you substitute `INPUT_PATH` — do not leave the literal `IS_DIRECTED` in the code.
|
||||
|
||||
```bash
|
||||
mkdir -p graphify-out
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from graphify.build import build_from_json
|
||||
from graphify.cluster import cluster, score_all
|
||||
from graphify.analyze import god_nodes, surprising_connections, suggest_questions
|
||||
from graphify.report import generate
|
||||
from graphify.export import to_json
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
|
||||
# root= mirrors the --update runbook (#1361): relativize source_file to the same
|
||||
# base so the full build and incremental --update never drift apart on re-extract.
|
||||
G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED)
|
||||
# Guard BEFORE any write: an empty extraction must not clobber a good graph.json /
|
||||
# GRAPH_REPORT.md / analysis sidecar. Check immediately after build (#1392).
|
||||
if G.number_of_nodes() == 0:
|
||||
print('ERROR: Graph is empty - extraction produced no nodes.')
|
||||
print('Possible causes: all files were skipped, binary-only corpus, or extraction failed.')
|
||||
raise SystemExit(1)
|
||||
communities = cluster(G)
|
||||
cohesion = score_all(G, communities)
|
||||
tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)}
|
||||
gods = god_nodes(G)
|
||||
surprises = surprising_connections(G, communities)
|
||||
labels = {cid: 'Community ' + str(cid) for cid in communities}
|
||||
# Placeholder questions - regenerated with real labels in Step 5
|
||||
questions = suggest_questions(G, communities, labels)
|
||||
|
||||
# Export FIRST and honor the #479 shrink-guard: to_json returns False (writing
|
||||
# nothing) when the new graph is smaller than the existing graph.json. Only write
|
||||
# GRAPH_REPORT.md + the analysis sidecar when the graph was actually written, so
|
||||
# they never describe a graph that graph.json doesn't contain (#1392).
|
||||
wrote = to_json(G, communities, 'graphify-out/graph.json')
|
||||
if not wrote:
|
||||
print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).')
|
||||
print('If this shrink is intentional (you deleted files), re-run a full build with --force.')
|
||||
raise SystemExit(1)
|
||||
report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, 'INPUT_PATH', suggested_questions=questions)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\")
|
||||
analysis = {
|
||||
'communities': {str(k): v for k, v in communities.items()},
|
||||
'cohesion': {str(k): v for k, v in cohesion.items()},
|
||||
'gods': gods,
|
||||
'surprises': surprises,
|
||||
'questions': questions,
|
||||
}
|
||||
Path('graphify-out/.graphify_analysis.json').write_text(json.dumps(analysis, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges, {len(communities)} communities')
|
||||
"
|
||||
```
|
||||
|
||||
If this step prints `ERROR: Graph is empty`, stop and tell the user what happened - do not proceed to labeling or visualization.
|
||||
|
||||
Replace INPUT_PATH with the actual path.
|
||||
|
||||
### Step 4.5 - Graph health check (read-only integrity gate)
|
||||
|
||||
A non-destructive diagnostic on the extraction, before labeling. It surfaces edge collapse, dangling/missing endpoints, and self-loops — the silent-corruption modes of incremental updates and AST/LLM id mismatches. Read-only; never aborts.
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
from graphify.diagnostics import diagnose_extraction, format_diagnostic_report
|
||||
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
summary = diagnose_extraction(extraction, directed=IS_DIRECTED, root='INPUT_PATH')
|
||||
print(format_diagnostic_report(summary))
|
||||
flags = [f'{summary[k]} {label}' for k, label in (
|
||||
('dangling_endpoint_edges', 'dangling-endpoint edges'),
|
||||
('missing_endpoint_edges', 'missing-endpoint edges'),
|
||||
('self_loop_edges', 'self-loop edges'),
|
||||
('directed_same_endpoint_collapsed_edges', 'collapsed (directed) edges'),
|
||||
('undirected_same_endpoint_collapsed_edges', 'collapsed (undirected) edges'),
|
||||
) if summary.get(k, 0)]
|
||||
print('GRAPH HEALTH WARNING: ' + '; '.join(flags) + ' - graph may be incomplete/corrupt.' if flags else 'Graph health: OK (no dangling/missing/collapsed edges).')
|
||||
"
|
||||
```
|
||||
|
||||
Substitute `IS_DIRECTED` and `INPUT_PATH` as in Step 4. If a `GRAPH HEALTH WARNING` prints, surface it in the final summary (do not abort — the graph is still usable, but the integrity issue must be visible, per the Honesty Rules).
|
||||
|
||||
### Step 5 - Label communities
|
||||
|
||||
Read `graphify-out/.graphify_analysis.json`. For each community key, look at its node labels and write a 2-5 word plain-language name (e.g. "Attention Mechanism", "Training Pipeline", "Data Loading").
|
||||
|
||||
Then regenerate the report and save the labels for the visualizer:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from graphify.build import build_from_json
|
||||
from graphify.cluster import score_all
|
||||
from graphify.analyze import god_nodes, surprising_connections, suggest_questions
|
||||
from graphify.report import generate
|
||||
from graphify.export import to_json
|
||||
from pathlib import Path
|
||||
|
||||
extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
detection = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
analysis = json.loads(Path('graphify-out/.graphify_analysis.json').read_text(encoding=\"utf-8\"))
|
||||
|
||||
# root= as in Step 4 / the --update runbook (#1361) — same base for node-key parity.
|
||||
G = build_from_json(extraction, root='INPUT_PATH', directed=IS_DIRECTED)
|
||||
communities = {int(k): v for k, v in analysis['communities'].items()}
|
||||
cohesion = {int(k): v for k, v in analysis['cohesion'].items()}
|
||||
tokens = {'input': extraction.get('input_tokens', 0), 'output': extraction.get('output_tokens', 0)}
|
||||
|
||||
# LABELS - replace these with the names you chose above
|
||||
labels = LABELS_DICT
|
||||
|
||||
# Regenerate questions with real community labels (labels affect question phrasing)
|
||||
questions = suggest_questions(G, communities, labels)
|
||||
|
||||
report = generate(G, communities, cohesion, labels, analysis['gods'], analysis['surprises'], detection, tokens, 'INPUT_PATH', suggested_questions=questions)
|
||||
Path('graphify-out/GRAPH_REPORT.md').write_text(report, encoding=\"utf-8\")
|
||||
Path('graphify-out/.graphify_labels.json').write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
# Re-export so graph.json nodes carry the curated community_name (#2490).
|
||||
# Same extraction as Step 4, so the #479 shrink-guard passes on node count;
|
||||
# if it still refuses, surface the guard message - do not force past it.
|
||||
wrote = to_json(G, communities, 'graphify-out/graph.json', community_labels=labels)
|
||||
if not wrote:
|
||||
print('ERROR: refused to shrink graphify-out/graph.json (existing graph has more nodes; #479).')
|
||||
print('If this shrink is intentional (you deleted files), re-run a full build with --force.')
|
||||
print('Report updated with community labels')
|
||||
"
|
||||
```
|
||||
|
||||
Replace `LABELS_DICT` with the actual dict you constructed (e.g. `{0: "Attention Mechanism", 1: "Training Pipeline"}`).
|
||||
Replace INPUT_PATH with the actual path.
|
||||
|
||||
### Step 6 - Generate Obsidian vault (opt-in) + HTML
|
||||
|
||||
**Generate HTML always** (unless `--no-viz`). **Obsidian vault only if `--obsidian` was explicitly given** — skip it otherwise, it generates one file per node.
|
||||
|
||||
If `--obsidian` was given:
|
||||
|
||||
- If `--obsidian-dir <path>` was also given, pass it via `--dir`. Otherwise defaults to `graphify-out/obsidian`.
|
||||
|
||||
```bash
|
||||
graphify export obsidian
|
||||
# or with custom dir: graphify export obsidian --dir ~/vaults/my-project
|
||||
```
|
||||
|
||||
Generate the HTML graph (always, unless `--no-viz`):
|
||||
|
||||
```bash
|
||||
graphify export html # auto-aggregates to community view if graph > 5000 nodes
|
||||
# or: graphify export html --no-viz
|
||||
```
|
||||
|
||||
### Steps 6b-8 - Wiki, Neo4j, FalkorDB, SVG, GraphML, MCP, benchmark (only on their flags)
|
||||
|
||||
These run only when their flag is present (`--wiki`, `--neo4j`/`--neo4j-push`, `--falkordb`/`--falkordb-push`, `--svg`, `--graphml`, `--mcp`) or, for the token-reduction benchmark, when `total_words` exceeds 5,000. A default run with no export flags skips all of them. See `references/exports.md` for each one. Run any `--wiki` export before Step 9 cleanup so `.graphify_labels.json` is still available.
|
||||
|
||||
---
|
||||
|
||||
### Step 9 - Save manifest, update cost tracker, clean up, and report
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from graphify.detect import save_manifest
|
||||
|
||||
# Save manifest for --update
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
# In --update mode, 'all_files' carries the full corpus; 'files' is the changed
|
||||
# subset. Full-rebuild mode populates only 'files', so the fallback handles that.
|
||||
# root= relativizes the manifest keys to the scan root (same base as the build),
|
||||
# so the on-disk manifest is portable across clones/machines and a later --update
|
||||
# matches cached files instead of missing every one (#1417).
|
||||
#
|
||||
# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output:
|
||||
# a detected file whose chunk failed or was omitted must stay unstamped so the
|
||||
# next --update re-queues it, otherwise it is marked done and its content is lost
|
||||
# forever (#2015). This mirrors the library extract path exactly
|
||||
# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the
|
||||
# raw corpus. Code files are always stamped (AST is deterministic); only semantic
|
||||
# types are gated on output.
|
||||
from graphify.cli import _stamped_manifest_files
|
||||
_corpus = detect.get('all_files') or detect['files']
|
||||
# A file a subagent only partly read DID produce output, so the stamp filter alone would
|
||||
# keep it - but its node set covers just the head of the file, and stamping it means
|
||||
# detect_incremental reads it as done and never re-queues it, leaving the truncated graph
|
||||
# live forever. Name those files so they stay unstamped and are retried next run (#2225).
|
||||
_pf = Path('graphify-out/.graphify_partial.txt')
|
||||
_partial = {l.strip() for l in _pf.read_text(encoding=\"utf-8\").splitlines() if l.strip()} if _pf.exists() else set()
|
||||
_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH'), partial_source_files=_partial)
|
||||
# Files dispatched this run (the changed subset) but NOT stamped above still carry
|
||||
# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues
|
||||
# them instead of reading them as unchanged (#1948).
|
||||
_sem_types = ('document', 'paper', 'image')
|
||||
_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl}
|
||||
_stamped = {f for fl in _manifest_files.values() for f in fl}
|
||||
_cleared = _dispatched - _stamped
|
||||
# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root
|
||||
# files newly excluded since last run are dropped rather than masquerading as
|
||||
# deletions; untouched files' prior rows are still preserved (#1908).
|
||||
_scan = {f for fl in _corpus.values() for f in fl}
|
||||
save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None)
|
||||
|
||||
# Update cumulative cost tracker
|
||||
input_tok = extract.get('input_tokens', 0)
|
||||
output_tok = extract.get('output_tokens', 0)
|
||||
|
||||
cost_path = Path('graphify-out/cost.json')
|
||||
if cost_path.exists():
|
||||
cost = json.loads(cost_path.read_text(encoding=\"utf-8\"))
|
||||
else:
|
||||
cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0}
|
||||
|
||||
cost['runs'].append({
|
||||
'date': datetime.now(timezone.utc).isoformat(),
|
||||
'input_tokens': input_tok,
|
||||
'output_tokens': output_tok,
|
||||
'files': detect.get('total_files', 0),
|
||||
})
|
||||
cost['total_input_tokens'] += input_tok
|
||||
cost['total_output_tokens'] += output_tok
|
||||
cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
|
||||
print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens')
|
||||
print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)')
|
||||
"
|
||||
rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_partial.txt
|
||||
find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null
|
||||
rm -f graphify-out/.needs_update 2>/dev/null || true
|
||||
```
|
||||
|
||||
Replace INPUT_PATH with the actual path (same value used in Steps 4-5) so the manifest is relativized to the scan root.
|
||||
|
||||
Tell the user (omit the obsidian line unless --obsidian was given):
|
||||
```
|
||||
Graph complete. Outputs in PATH_TO_DIR/graphify-out/
|
||||
|
||||
graph.html - interactive graph, open in browser
|
||||
GRAPH_REPORT.md - audit report
|
||||
graph.json - raw graph data
|
||||
obsidian/ - Obsidian vault (only if --obsidian was given)
|
||||
```
|
||||
|
||||
If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi
|
||||
|
||||
Replace PATH_TO_DIR with the actual absolute path of the directory that was processed.
|
||||
|
||||
If PATH_TO_DIR is inside a git repository, add one line to the report: `graphify-out/` was written into the working tree, so unless the repository already ignores it, it will show in `git status`; the user can add `graphify-out/` to `.gitignore`, or commit only the outputs their team should share (see https://docs.graphify.com/guides/team-workflows).
|
||||
|
||||
Then paste these sections from GRAPH_REPORT.md directly into the chat:
|
||||
- God Nodes
|
||||
- Surprising Connections
|
||||
- Suggested Questions
|
||||
|
||||
Do NOT paste the full report - just those three sections. Keep it concise.
|
||||
|
||||
Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask:
|
||||
|
||||
> "The most interesting question this graph can answer: **[question]**. Want me to trace it?"
|
||||
|
||||
If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report.
|
||||
|
||||
The graph is the map. Your job after the pipeline is to be the guide.
|
||||
|
||||
---
|
||||
|
||||
## Interpreter guard for subcommands
|
||||
|
||||
Before running any subcommand below (`--update`, `--cluster-only`, `query`, `path`, `explain`, `add`), check that `.graphify_python` exists. If it's missing (e.g. user deleted `graphify-out/`), re-resolve the interpreter first:
|
||||
|
||||
```bash
|
||||
if [ ! -f graphify-out/.graphify_python ]; then
|
||||
GRAPHIFY_BIN=$(which graphify 2>/dev/null)
|
||||
if [ -n "$GRAPHIFY_BIN" ]; then
|
||||
PYTHON=$(head -1 "$GRAPHIFY_BIN" | tr -d '#!')
|
||||
case "$PYTHON" in *[!a-zA-Z0-9/_.@-]*) PYTHON="python3" ;; esac
|
||||
else
|
||||
PYTHON="python3"
|
||||
fi
|
||||
mkdir -p graphify-out
|
||||
"$PYTHON" -c "import sys; open('graphify-out/.graphify_python', 'w', encoding='utf-8').write(sys.executable)"
|
||||
fi
|
||||
```
|
||||
|
||||
## For --update and --cluster-only
|
||||
|
||||
Both are non-default subcommands. `--update` re-extracts only new or changed files; `--cluster-only` reruns clustering on the existing graph. See `references/update.md` for both flows.
|
||||
|
||||
---
|
||||
|
||||
## For /graphify query
|
||||
|
||||
When `graphify-out/graph.json` already exists and the user asks a question about the corpus, answer from the graph rather than rebuilding it:
|
||||
|
||||
```bash
|
||||
graphify query "<question>"
|
||||
```
|
||||
|
||||
Before traversal, expand the question against the graph's own vocabulary so a wording mismatch does not collapse the answer to noise. If the `graphify query` CLI is unavailable, fall back to an inline NetworkX traversal of `graphify-out/graph.json`. Answer using only what the graph output contains, and quote `source_location` when citing a specific fact. For that vocab-expansion step, the BFS/DFS traversal modes, the `--budget` cap, the NetworkX fallback, `save-result` feedback, and the `/graphify path` and `/graphify explain` flows, see `references/query.md`.
|
||||
|
||||
---
|
||||
|
||||
## For /graphify add and --watch
|
||||
|
||||
Neither is part of the default build. When the user runs `/graphify add <url>` to fetch a URL into the corpus, or passes `--watch` to auto-rebuild on file changes, see `references/add-watch.md`.
|
||||
|
||||
---
|
||||
|
||||
## For the commit hook and native CLAUDE.md integration
|
||||
|
||||
When the user asks to install the post-commit auto-rebuild hook or wire graphify into a project's CLAUDE.md, see `references/hooks.md`.
|
||||
|
||||
---
|
||||
|
||||
## Honesty Rules
|
||||
|
||||
- Never invent an edge. If unsure, use AMBIGUOUS.
|
||||
- Never skip the corpus check warning.
|
||||
- Always show token cost in the report.
|
||||
- Never hide cohesion scores behind symbols - show the raw number.
|
||||
- Never run HTML viz on a graph with more than 5,000 nodes without warning the user.
|
||||
@@ -0,0 +1,56 @@
|
||||
# graphify reference: add a URL and watch a folder
|
||||
|
||||
Load this when the user ran `/graphify add <url>` or passed `--watch`. Neither is part of the default build.
|
||||
|
||||
## For /graphify add
|
||||
|
||||
Fetch a URL and add it to the corpus, then update the graph.
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys
|
||||
from graphify.ingest import ingest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
out = ingest('URL', Path('./raw'), author='AUTHOR', contributor='CONTRIBUTOR')
|
||||
print(f'Saved to {out}')
|
||||
except ValueError as e:
|
||||
print(f'error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except RuntimeError as e:
|
||||
print(f'error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
"
|
||||
```
|
||||
|
||||
Replace `URL` with the actual URL, `AUTHOR` with the user's name if provided, `CONTRIBUTOR` likewise. If the command exits with an error, tell the user what went wrong - do not silently continue. After a successful save, automatically run the `--update` pipeline on `./raw` to merge the new file into the existing graph.
|
||||
|
||||
Supported URL types (auto-detected):
|
||||
- YouTube / any video URL → audio downloaded via yt-dlp, transcribed to `.txt` on next run (requires `pip install 'graphifyy[video]'`)
|
||||
- Twitter/X → fetched via oEmbed, saved as `.md` with tweet text and author
|
||||
- arXiv → abstract + metadata saved as `.md`
|
||||
- PDF → downloaded as `.pdf`
|
||||
- Images (.png/.jpg/.webp) → downloaded, Claude vision extracts on next run
|
||||
- Any webpage → converted to markdown via html2text
|
||||
|
||||
---
|
||||
|
||||
## For --watch
|
||||
|
||||
Start a background watcher that monitors a folder and auto-updates the graph when files change.
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -m graphify.watch "$(cat graphify-out/.graphify_root)" --debounce 3
|
||||
```
|
||||
|
||||
This watches the same folder graphify extracted, read from the trusted `graphify-out/.graphify_root` that Step 1 resolved - there is no path to substitute, so a scan root containing shell metacharacters can never be re-interpreted here. Behavior depends on what changed:
|
||||
|
||||
- **Code files only (.py, .ts, .go, etc.):** re-runs AST extraction + rebuild + cluster immediately, no LLM needed. `graph.json` and `GRAPH_REPORT.md` are updated automatically.
|
||||
- **Docs, papers, or images:** writes a `graphify-out/needs_update` flag and prints a notification to run `/graphify --update` (LLM semantic re-extraction required).
|
||||
|
||||
Debounce (default 3s): waits until file activity stops before triggering, so a wave of parallel agent writes doesn't trigger a rebuild per file.
|
||||
|
||||
Press Ctrl+C to stop.
|
||||
|
||||
For agentic workflows: run `--watch` in a background terminal. Code changes from agent waves are picked up automatically between waves. If agents are also writing docs or notes, you'll need a manual `/graphify --update` after those waves.
|
||||
@@ -0,0 +1,91 @@
|
||||
# graphify reference: extra exports and benchmark
|
||||
|
||||
Load this when the user passed one of the export flags (`--wiki`, `--neo4j`, `--neo4j-push`, `--falkordb`, `--falkordb-push`, `--svg`, `--graphml`, `--mcp`), or when the corpus is large enough for the token-reduction benchmark. Each step runs only for its own flag.
|
||||
|
||||
### Step 6b - Wiki (only if --wiki flag)
|
||||
|
||||
**Only run this step if `--wiki` was explicitly given in the original command.**
|
||||
|
||||
Run this before Step 9 (cleanup) so `.graphify_labels.json` is still available.
|
||||
|
||||
```bash
|
||||
graphify export wiki
|
||||
```
|
||||
|
||||
### Step 7 - Neo4j export (only if --neo4j or --neo4j-push flag)
|
||||
|
||||
**If `--neo4j`** - generate a Cypher file for manual import:
|
||||
|
||||
```bash
|
||||
graphify export neo4j
|
||||
```
|
||||
|
||||
**If `--neo4j-push`** - push directly to Neo4j using the destination URI supplied with the original `/graphify` command. Only push when this flag was explicitly given.
|
||||
|
||||
Pass the exact URI from the original `/graphify` command as one argument. When constructing a shell command, quote the URI for the shell that will execute the command (use `shlex.quote` for POSIX shells), or use a structured argument interface. Keep the URI as data; never interpolate it unquoted or evaluate it as shell code. Never replace a supplied URI with localhost. If the user explicitly requested a push but supplied no URI, use the documented local default. Do not print the URI or credentials in status output or generated artifacts.
|
||||
|
||||
```bash
|
||||
graphify export neo4j --push 'bolt://remote-host:7687' --user neo4j
|
||||
```
|
||||
|
||||
Default URI is `bolt://localhost:7687`, default user is `neo4j`. Prefer the `NEO4J_PASSWORD` environment variable so the password is kept off argv; ask for credentials only when needed. Uses MERGE - safe to re-run without creating duplicates.
|
||||
|
||||
### Step 7a - FalkorDB export (only if --falkordb or --falkordb-push flag)
|
||||
|
||||
**If `--falkordb`** - generate a Cypher file. The statements are OpenCypher, but FalkorDB's `GRAPH.QUERY` runs one statement at a time (no bulk script import like Neo4j's `cypher-shell`), so prefer `--falkordb-push` to load a graph. Use this only when you want the portable `cypher.txt` artifact:
|
||||
|
||||
```bash
|
||||
graphify export falkordb
|
||||
```
|
||||
|
||||
**If `--falkordb-push`** - push directly to FalkorDB using the destination URI supplied with the original `/graphify` command. Only push when this flag was explicitly given.
|
||||
|
||||
Pass the exact URI from the original `/graphify` command as one argument. When constructing a shell command, quote the URI for the shell that will execute the command (use `shlex.quote` for POSIX shells), or use a structured argument interface. Keep the URI as data; never interpolate it unquoted or evaluate it as shell code. Never replace a supplied URI with localhost. If the user explicitly requested a push but supplied no URI, use the documented local default. Do not print the URI or credentials in status output or generated artifacts. Credentials are optional; ask only if the instance requires authentication, and use `FALKORDB_PASSWORD` for a password rather than printing it.
|
||||
|
||||
```bash
|
||||
graphify export falkordb --push 'falkordb://remote-host:6379'
|
||||
```
|
||||
|
||||
Default URI is `falkordb://localhost:6379` (the scheme is informational - `redis://` or a bare `host:port` work too), auth is optional, and the target graph defaults to `graphify`. Uses MERGE - safe to re-run without creating duplicates.
|
||||
|
||||
### Step 7b - SVG export (only if --svg flag)
|
||||
|
||||
```bash
|
||||
graphify export svg
|
||||
```
|
||||
|
||||
### Step 7c - GraphML export (only if --graphml flag)
|
||||
|
||||
```bash
|
||||
graphify export graphml
|
||||
```
|
||||
|
||||
### Step 7d - MCP server (only if --mcp flag)
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -m graphify.serve graphify-out/graph.json
|
||||
```
|
||||
|
||||
This starts a stdio MCP server that exposes tools: `query_graph`, `get_node`, `get_neighbors`, `get_community`, `god_nodes`, `graph_stats`, `shortest_path`. Add to Claude Desktop or any MCP-compatible agent orchestrator so other agents can query the graph live.
|
||||
|
||||
To configure in Claude Desktop, add to `claude_desktop_config.json`. Claude Desktop can't run `$(...)`, and under `uv tool install` the system `python3` can't import graphify — so set `command` to the **absolute interpreter path** printed by `cat graphify-out/.graphify_python`:
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"graphify": {
|
||||
"command": "<absolute path from: cat graphify-out/.graphify_python>",
|
||||
"args": ["-m", "graphify.serve", "/absolute/path/to/graphify-out/graph.json"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Step 8 - Token reduction benchmark (only if total_words > 5000)
|
||||
|
||||
If `total_words` from `graphify-out/.graphify_detect.json` is greater than 5,000, run:
|
||||
|
||||
```bash
|
||||
graphify benchmark
|
||||
```
|
||||
|
||||
Print the output directly in chat. If `total_words <= 5000`, skip silently - the graph value is structural clarity, not token compression, for small corpora.
|
||||
@@ -0,0 +1,86 @@
|
||||
# graphify reference: extraction subagent prompt
|
||||
|
||||
Load this in Step 3 Part B when the corpus has at least one doc, paper, or image chunk. A pure-code corpus skips Part B and never reads this file. Each semantic subagent receives the prompt below verbatim (substitute FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH).
|
||||
|
||||
```
|
||||
You are a graphify extraction subagent. Read the files listed and extract a knowledge graph fragment.
|
||||
Output ONLY valid JSON matching the schema below - no explanation, no markdown fences, no preamble.
|
||||
|
||||
Files (chunk CHUNK_NUM of TOTAL_CHUNKS):
|
||||
FILE_LIST
|
||||
|
||||
Read every file in this list COMPLETELY before extracting anything from it. The Read tool
|
||||
returns at most 2000 lines per call and reports NO error and NO warning when it truncates, so a
|
||||
long file silently yields a partial extraction. When a file is longer than one call, keep paging
|
||||
with offset/limit until you have actually seen its last line - never assume the first call
|
||||
returned the whole file. Never emit nodes for a file you have only partly read. A partial
|
||||
extraction is worse than none: replace-on-re-extract first DROPS that file's existing nodes and
|
||||
then re-adds only what you saw, so the unread tail disappears from the graph, and the file is then
|
||||
recorded as successfully extracted and never retried (#2225).
|
||||
|
||||
Report the coverage you actually achieved, so a short read is caught instead of guessed at: add a
|
||||
top-level "coverage" object mapping each FILE_LIST path you extracted to the number of lines you
|
||||
read, e.g. {"coverage": {"docs/plan.md": 452}}. If you could not reach a file's last line, still
|
||||
report the count you did read. A short count is recoverable - Step B3 warns, the entry is cached
|
||||
as partial, and the next --update retries the file - whereas a missing entry is indistinguishable
|
||||
from a complete read.
|
||||
|
||||
Rules:
|
||||
- EXTRACTED: relationship explicit in source (import, call, citation, "see §3.2")
|
||||
- INFERRED: reasonable inference (shared data structure, implied dependency)
|
||||
- AMBIGUOUS: uncertain - flag for review, do not omit
|
||||
|
||||
Code files: focus on semantic edges AST cannot find (call relationships, shared data, arch patterns).
|
||||
Do not re-extract imports - AST already has those.
|
||||
Doc/paper files: extract named concepts, entities, citations. For rationale (WHY decisions were made, trade-offs, design intent): store as a `rationale` attribute on the relevant concept node — do NOT create a separate rationale node or fragment node. Only create a node for something that is itself a named entity or concept. Use `file_type:"rationale"` for concept-like nodes (ideas, principles, mechanisms, design patterns). `file_type` MUST be one of exactly these six values: `code`, `document`, `paper`, `image`, `rationale`, `concept`. Any other value is invalid and will be rejected.
|
||||
Code files: when adding `calls` edges, source MUST be the caller (the function/class doing the calling), target MUST be the callee. Never reverse this direction. `calls` edges MUST stay within one language: a Python function cannot `calls` a JS/TS/Go/Rust/Java symbol and vice versa — cross-language call edges are phantom artifacts, never emit them.
|
||||
Image files: use vision to understand what the image IS - do not just OCR.
|
||||
UI screenshot: layout patterns, design decisions, key elements, purpose.
|
||||
Chart: metric, trend/insight, data source.
|
||||
Tweet/post: claim as node, author, concepts mentioned.
|
||||
Diagram: components and connections.
|
||||
Research figure: what it demonstrates, method, result.
|
||||
Handwritten/whiteboard: ideas and arrows, mark uncertain readings AMBIGUOUS.
|
||||
|
||||
DEEP_MODE (if --mode deep was given): be aggressive with INFERRED edges - indirect deps,
|
||||
shared assumptions, latent couplings. Mark uncertain ones AMBIGUOUS instead of omitting.
|
||||
|
||||
Semantic similarity: if two concepts in this chunk solve the same problem or represent the same idea without any structural link (no import, no call, no citation), add a `semantically_similar_to` edge marked INFERRED with a confidence_score reflecting how similar they are (0.6-0.95). Examples:
|
||||
- Two functions that both validate user input but never call each other
|
||||
- A class in code and a concept in a paper that describe the same algorithm
|
||||
- Two error types that handle the same failure mode differently
|
||||
Only add these when the similarity is genuinely non-obvious and cross-cutting. Do not add them for trivially similar things.
|
||||
|
||||
Hyperedges: if 3 or more nodes clearly participate together in a shared concept, flow, or pattern that is not captured by pairwise edges alone, add a hyperedge to a top-level `hyperedges` array. Examples:
|
||||
- All classes that implement a common protocol or interface
|
||||
- All functions in an authentication flow (even if they don't all call each other)
|
||||
- All concepts from a paper section that form one coherent idea
|
||||
Use sparingly — only when the group relationship adds information beyond the pairwise edges. Maximum 3 hyperedges per chunk.
|
||||
|
||||
If a file has YAML frontmatter (--- ... ---), copy source_url, captured_at, author,
|
||||
contributor onto every node from that file.
|
||||
|
||||
confidence_score is REQUIRED on every edge - never omit it, never use 0.5 as a default:
|
||||
- EXTRACTED edges: confidence_score = 1.0 always
|
||||
- INFERRED edges: pick exactly ONE value from this set — never 0.5:
|
||||
0.95 direct structural evidence (shared data structure, named cross-file reference).
|
||||
0.85 strong inference (clear functional alignment, no direct symbol link).
|
||||
0.75 reasonable inference (shared problem domain + similar shape, requires interpretation).
|
||||
0.65 weak inference (thematically related, no shape evidence).
|
||||
0.55 speculative but plausible (surface-level co-occurrence only).
|
||||
Models follow discrete rubrics better than continuous ranges; the bimodal
|
||||
distribution observed in production (>50% at 0.5, >40% at 0.85+) shows the
|
||||
range guidance is being collapsed to a binary. If no value above fits, mark
|
||||
the edge AMBIGUOUS rather than picking 0.4 or below.
|
||||
- AMBIGUOUS edges: 0.1-0.3
|
||||
|
||||
Node ID format: lowercase, word characters only (`[^\W]` after casefolding — `[a-z0-9_]` for ASCII input, but non-ASCII letters and digits are kept as-is), no dots or slashes. Format: `{stem}_{entity}` where stem is the **full repo-relative path with the extension dropped**, every path segment kept and joined with `_` (each segment lowercased with non-alphanumeric chars replaced by `_`), and entity is the symbol name similarly normalized. Use every directory level, not just the immediate parent — this keeps same-named files in different directories distinct. Examples: `src/auth/session.py` + `ValidateToken` → `src_auth_session_validatetoken`; `lib/utils/helpers.py` + `parse_url` → `lib_utils_helpers_parse_url`; `tests/test_foo.py` + `_helper` → `tests_test_foo_helper`; `docs/v1/api/README.md` + `getUser` → `docs_v1_api_readme_getuser`. Top-level files (no parent dir, e.g. `setup.py`) use just the filename stem: `setup_my_func`. This must match the ID the AST extractor generates — using just the filename (e.g., `session_validatetoken`) or only the immediate parent (e.g., `auth_session_validatetoken`) will create orphan ghost-duplicate nodes. If you are re-extracting a project built under the old immediate-parent format, the user should run `graphify extract --force` to rebuild cleanly. CRITICAL: never append chunk numbers, sequence numbers, or any suffix to an ID (no `_c1`, `_c2`, `_chunk2`, etc.). IDs must be deterministic from the label alone — the same entity must always produce the same ID regardless of which chunk processes it.
|
||||
|
||||
Generate the extraction JSON matching this schema exactly:
|
||||
{"nodes":[{"id":"auth_session_validatetoken","label":"Human Readable Name","file_type":"code|document|paper|image|rationale|concept","source_file":"<FILE_LIST path verbatim>","source_location":null,"source_url":null,"captured_at":null,"author":null,"contributor":null}],"edges":[{"source":"node_id","target":"node_id","relation":"calls|implements|references|cites|conceptually_related_to|shares_data_with|semantically_similar_to|rationale_for","confidence":"EXTRACTED|INFERRED|AMBIGUOUS","confidence_score":1.0,"source_file":"<FILE_LIST path verbatim>","source_location":null,"weight":1.0}],"hyperedges":[{"id":"snake_case_id","label":"Human Readable Label","nodes":["node_id1","node_id2","node_id3"],"relation":"participate_in|implement|form","confidence":"EXTRACTED|INFERRED","confidence_score":0.75,"source_file":"<FILE_LIST path verbatim>"}],"input_tokens":0,"output_tokens":0,"coverage":{"<FILE_LIST path verbatim>":0}}
|
||||
|
||||
source_file RULE (every node, edge, and hyperedge): set source_file to the path of the originating file EXACTLY as it appears in FILE_LIST — verbatim and absolute. Do NOT shorten to a basename, do NOT re-relativize, do NOT strip any directory prefix, and do NOT change separators (the engine canonicalizes separators and relativizes against the build root downstream). Copy the FILE_LIST entry character-for-character. This keeps the full build and incremental --update on the same base, so build_merge's replace-on-re-extract matches the existing node instead of accumulating a duplicate.
|
||||
|
||||
Then write the JSON to disk using the Write tool at this exact absolute path (no relative paths — Write resolves relative paths against an undefined cwd and the file will be silently lost):
|
||||
CHUNK_PATH
|
||||
```
|
||||
@@ -0,0 +1,46 @@
|
||||
# graphify reference: GitHub clone and cross-repo merge
|
||||
|
||||
Load this when the user passed one or more `https://github.com/...` URLs, or named several local subfolders to merge into one graph.
|
||||
|
||||
### Step 0 - Clone GitHub repo(s) (only if a GitHub URL was given)
|
||||
|
||||
**Single repo:**
|
||||
```bash
|
||||
LOCAL_PATH=$(graphify clone <github-url> [--branch <branch>])
|
||||
# Use LOCAL_PATH as the target for all subsequent steps
|
||||
```
|
||||
|
||||
**Multiple repos (cross-repo graph):**
|
||||
```bash
|
||||
# Clone each repo, run the full pipeline on each, then merge
|
||||
graphify clone <url1> # → ~/.graphify/repos/<owner1>/<repo1>
|
||||
graphify clone <url2> # → ~/.graphify/repos/<owner2>/<repo2>
|
||||
# Run /graphify on each local path to produce their graph.json files
|
||||
# Then merge:
|
||||
graphify merge-graphs \
|
||||
~/.graphify/repos/<owner1>/<repo1>/graphify-out/graph.json \
|
||||
~/.graphify/repos/<owner2>/<repo2>/graphify-out/graph.json \
|
||||
--out graphify-out/cross-repo-graph.json
|
||||
```
|
||||
|
||||
Graphify clones into `~/.graphify/repos/<owner>/<repo>` and reuses existing clones on repeat runs. Each node in the merged graph carries a `repo` attribute so you can filter by origin.
|
||||
|
||||
**Multiple local subfolders (monorepo or multi-service layout):**
|
||||
|
||||
The skill pipeline writes all intermediate and final outputs to `graphify-out/` in the current working directory. Running the skill on each subfolder separately will clobber the same output dir. Instead, use the CLI directly for each subfolder — it places `graphify-out/` *inside* the scanned path:
|
||||
|
||||
```bash
|
||||
graphify extract ./core/ # → ./core/graphify-out/graph.json
|
||||
graphify extract ./service/ # → ./service/graphify-out/graph.json
|
||||
graphify extract ./platform/ # → ./platform/graphify-out/graph.json
|
||||
# Add --backend gemini|kimi|openai|deepseek|claude-cli depending on which API key you have set
|
||||
|
||||
# Then merge at the project root:
|
||||
graphify merge-graphs \
|
||||
./core/graphify-out/graph.json \
|
||||
./service/graphify-out/graph.json \
|
||||
./platform/graphify-out/graph.json \
|
||||
--out graphify-out/graph.json
|
||||
```
|
||||
|
||||
Once `graphify-out/graph.json` exists, the fast path above takes over: any codebase question runs `graphify query` directly on the merged graph — no re-extraction, no size gate.
|
||||
@@ -0,0 +1,33 @@
|
||||
# graphify reference: commit hook and native CLAUDE.md integration
|
||||
|
||||
Load this when the user asked to install the post-commit hook or wire graphify into a project's CLAUDE.md.
|
||||
|
||||
## For git commit hook
|
||||
|
||||
Install a post-commit hook that auto-rebuilds the graph after every commit. No background process needed - triggers once per commit, works with any editor.
|
||||
|
||||
```bash
|
||||
graphify hook install # install
|
||||
graphify hook uninstall # remove
|
||||
graphify hook status # check
|
||||
```
|
||||
|
||||
After every `git commit`, the hook detects which code files changed (via `git diff HEAD~1`), re-runs AST extraction on those files, and rebuilds `graph.json` and `GRAPH_REPORT.md`. Doc/image changes are ignored by the hook - run `/graphify --update` manually for those.
|
||||
|
||||
If a post-commit hook already exists, graphify appends to it rather than replacing it.
|
||||
|
||||
---
|
||||
|
||||
## For native CLAUDE.md integration
|
||||
|
||||
Run once per project to make graphify always-on in Claude Code sessions:
|
||||
|
||||
```bash
|
||||
graphify claude install
|
||||
```
|
||||
|
||||
This writes a `## graphify` section to the local `CLAUDE.md` that instructs Claude to check the graph before answering codebase questions and rebuild it after code changes. No manual `/graphify` needed in future sessions.
|
||||
|
||||
```bash
|
||||
graphify claude uninstall # remove the section
|
||||
```
|
||||
@@ -0,0 +1,317 @@
|
||||
# graphify reference: query, path, explain
|
||||
|
||||
Load this when the user asks a question against an existing graph, or runs `/graphify path` or `/graphify explain`. The core's query stub points here for the full traversal flow. These flows use the `graphify query` CLI when it is available and fall back to an inline NetworkX traversal otherwise.
|
||||
|
||||
Two traversal modes - choose based on the question:
|
||||
|
||||
| Mode | Flag | Best for |
|
||||
|------|------|----------|
|
||||
| BFS (default) | _(none)_ | "What is X connected to?" - broad context, nearest neighbors first |
|
||||
| DFS | `--dfs` | "How does X reach Y?" - trace a specific chain or dependency path |
|
||||
|
||||
First check the graph exists:
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
from pathlib import Path
|
||||
if not Path('graphify-out/graph.json').exists():
|
||||
print('ERROR: No graph found. Run /graphify <path> first to build the graph.')
|
||||
raise SystemExit(1)
|
||||
"
|
||||
```
|
||||
If it fails, stop and tell the user to run `/graphify <path>` first.
|
||||
|
||||
### Step 0 — Constrained query expansion (REQUIRED before traversal)
|
||||
|
||||
graphify's `query` CLI matches nodes via case-folded substring + IDF — there is **no stemming, no synonyms, no cross-language match** inside the binary, and the inline fallback below matches the same way. If the user's question uses different language or different domain vocabulary than the graph's labels (user says "обработчик" / graph says "handler"; user says "authentication" / graph says "Guardian"), the literal matcher returns 0 hits and the answer collapses to noise.
|
||||
|
||||
Fix this **without inventing tokens** by expanding the query against the actual graph vocabulary first:
|
||||
|
||||
1. Extract the token vocabulary from node labels:
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, re
|
||||
from pathlib import Path
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8'))
|
||||
vocab = set()
|
||||
for n in data['nodes']:
|
||||
for c in re.findall(r'[^\W\d_]+', n.get('label','') or '', re.UNICODE):
|
||||
parts = re.findall(r'[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+', c) or [c]
|
||||
for p in parts:
|
||||
t = p.lower()
|
||||
if 3 <= len(t) <= 30:
|
||||
vocab.add(t)
|
||||
Path('graphify-out/.vocab.txt').write_text('\n'.join(sorted(vocab)), encoding='utf-8')
|
||||
print(f'vocab: {len(vocab)} tokens')
|
||||
"
|
||||
```
|
||||
|
||||
2. Read `graphify-out/.vocab.txt`. Then for the user's question, select **up to 12 tokens from this exact list** that semantically match the query intent. Hard constraints:
|
||||
- You MUST pick only tokens present in the vocabulary file. Do NOT invent tokens.
|
||||
- If a query concept has no plausible token in the vocab, skip it — do not substitute a near-synonym from training memory.
|
||||
- If **no** vocab tokens match the query at all, output an empty list and tell the user the corpus has no relevant vocabulary for this question. Do not fabricate a search.
|
||||
- Translate cross-language: Russian "аутентификация" → look for `auth`, `credential`, `token`, `security` IFF present in vocab.
|
||||
- Morphology: "handlers" maps to `handler` IFF present; "todos" maps to `todo` IFF present.
|
||||
|
||||
3. Print the selection explicitly to the user before running the query, so the expansion is auditable:
|
||||
```
|
||||
Query expanded to (from graph vocab, N tokens): [token1, token2, ...]
|
||||
```
|
||||
If the list is empty, say so plainly and stop — do not proceed to traversal.
|
||||
|
||||
### Step 1 — Traversal
|
||||
|
||||
Build the **expanded query string** by joining the selected tokens with spaces. Use this string as `QUESTION` below — NOT the original user question. (The original question is preserved only for `save-result` at the end.)
|
||||
|
||||
Prefer the CLI when it is installed:
|
||||
```bash
|
||||
graphify query "QUESTION"
|
||||
# or: graphify query "QUESTION" --dfs --budget 3000
|
||||
```
|
||||
|
||||
If the CLI is unavailable, load `graphify-out/graph.json` and run the traversal inline:
|
||||
|
||||
1. Find the 1-3 nodes whose label best matches the expanded tokens.
|
||||
2. Run the appropriate traversal from each starting node.
|
||||
3. Read the subgraph - node labels, edge relations, confidence tags, source locations.
|
||||
4. Answer using **only** what the graph contains. Quote `source_location` when citing a specific fact.
|
||||
5. If the graph lacks enough information, say so - do not hallucinate edges.
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from networkx.readwrite import json_graph
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8'))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
question = 'QUESTION'
|
||||
mode = 'MODE' # 'bfs' or 'dfs'
|
||||
terms = [t.lower() for t in question.split() if len(t) >= 3] # match the vocab threshold; keeps api/jwt/ios (#1392)
|
||||
|
||||
# Find best-matching start nodes
|
||||
scored = []
|
||||
for nid, ndata in G.nodes(data=True):
|
||||
label = ndata.get('label', '').lower()
|
||||
score = sum(1 for t in terms if t in label)
|
||||
if score > 0:
|
||||
scored.append((score, nid))
|
||||
scored.sort(reverse=True)
|
||||
start_nodes = [nid for _, nid in scored[:3]]
|
||||
|
||||
if not start_nodes:
|
||||
print('No matching nodes found for query terms:', terms)
|
||||
sys.exit(0)
|
||||
|
||||
subgraph_nodes = set()
|
||||
|
||||
if mode == 'dfs':
|
||||
# DFS: follow one path as deep as possible before backtracking.
|
||||
# Depth-limited to 6 to avoid traversing the whole graph.
|
||||
visited = set()
|
||||
stack = [(n, 0) for n in reversed(start_nodes)]
|
||||
while stack:
|
||||
node, depth = stack.pop()
|
||||
if node in visited or depth > 6:
|
||||
continue
|
||||
visited.add(node)
|
||||
subgraph_nodes.add(node)
|
||||
for neighbor in G.neighbors(node):
|
||||
if neighbor not in visited:
|
||||
stack.append((neighbor, depth + 1))
|
||||
else:
|
||||
# BFS: explore all neighbors layer by layer up to depth 3.
|
||||
frontier = set(start_nodes)
|
||||
subgraph_nodes = set(start_nodes)
|
||||
for _ in range(3):
|
||||
next_frontier = set()
|
||||
for n in frontier:
|
||||
for neighbor in G.neighbors(n):
|
||||
if neighbor not in subgraph_nodes:
|
||||
next_frontier.add(neighbor)
|
||||
subgraph_nodes.update(next_frontier)
|
||||
frontier = next_frontier
|
||||
|
||||
# Traversal chooses the nodes; collect relationships separately so seed edges,
|
||||
# cycles, and cross-links are not lost just because both endpoints were visited.
|
||||
# Taking the induced subgraph also keeps edges to nodes beyond the depth limit
|
||||
# out of the result. data=True retains each edge's own attributes, including
|
||||
# distinct parallel relationships in a MultiGraph.
|
||||
subgraph_edges = [
|
||||
(u, v, edge_data)
|
||||
for u, v, edge_data in G.subgraph(subgraph_nodes).edges(data=True)
|
||||
if u != v
|
||||
]
|
||||
|
||||
# Token-budget aware output: rank by relevance, cut at budget (~4 chars/token)
|
||||
token_budget = BUDGET # default 2000
|
||||
char_budget = token_budget * 4
|
||||
|
||||
# Score each node by term overlap for ranked output
|
||||
def relevance(nid):
|
||||
label = G.nodes[nid].get('label', '').lower()
|
||||
return sum(1 for t in terms if t in label)
|
||||
|
||||
ranked_nodes = sorted(subgraph_nodes, key=relevance, reverse=True)
|
||||
|
||||
lines = [f'Traversal: {mode.upper()} | Start: {[G.nodes[n].get(\"label\",n) for n in start_nodes]} | {len(subgraph_nodes)} nodes']
|
||||
for nid in ranked_nodes:
|
||||
d = G.nodes[nid]
|
||||
lines.append(f' NODE {d.get(\"label\", nid)} [src={d.get(\"source_file\",\"\")} loc={d.get(\"source_location\",\"\")}]')
|
||||
for u, v, edge_data in subgraph_edges:
|
||||
if u in subgraph_nodes and v in subgraph_nodes:
|
||||
lines.append(f' EDGE {G.nodes[u].get(\"label\",u)} --{edge_data.get(\"relation\",\"\")} [{edge_data.get(\"confidence\",\"\")}]--> {G.nodes[v].get(\"label\",v)}')
|
||||
|
||||
output = '\n'.join(lines)
|
||||
if len(output) > char_budget:
|
||||
output = output[:char_budget] + f'\n... (truncated at ~{token_budget} token budget - use --budget N for more)'
|
||||
print(output)
|
||||
"
|
||||
```
|
||||
|
||||
Replace `QUESTION` with the **expanded** query string, `MODE` with `bfs` or `dfs`, and `BUDGET` with the token budget (default `2000`, or whatever `--budget N` specifies). Then answer based on the subgraph output above, using only what the graph contains.
|
||||
|
||||
After writing the answer, save it back into the graph so it improves future queries. Include the expanded tokens inside the `--answer` text (e.g. `"Expanded from original query via vocab: [tokens]. Then traversed..."`) so the next `--update` extracts the expansion history as a graph node:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -m graphify save-result --question "ORIGINAL_QUESTION" --answer "ANSWER" --type query --nodes NODE1 NODE2
|
||||
```
|
||||
|
||||
Replace `ORIGINAL_QUESTION` with the user's verbatim question, `ANSWER` with your full answer text (containing the expanded-token trace), `NODE1 NODE2` with the list of node labels you cited. This closes the feedback loop: the next `--update` will extract this Q&A as a node in the graph.
|
||||
|
||||
**Work memory (self-improving loop).** Add an `--outcome` so future sessions learn from this one — append `--outcome useful|dead_end|corrected` to the `save-result` command (and `--correction "the right answer"` when correcting):
|
||||
|
||||
- `useful` — the cited nodes answered the question well (they become *preferred sources*).
|
||||
- `dead_end` — the question/path led nowhere; don't re-derive it next time.
|
||||
- `corrected` — the saved answer was wrong; `--correction` records what was right.
|
||||
|
||||
At the **start** of graph work, refresh and read the lessons: run `graphify reflect --if-stale` (cheap, deterministic, no LLM; `--if-stale` makes it a no-op when `LESSONS.md` is already newer than every input, e.g. when the git hook just refreshed it), then read `graphify-out/reflections/LESSONS.md`. It lists **preferred sources** (start there), **known dead ends** (skip them), and prior **corrections**. Running `reflect` yourself keeps the lessons current even without the git hook installed; if the post-commit hook *is* installed, `--if-stale` means your session-start run costs almost nothing.
|
||||
|
||||
---
|
||||
|
||||
## For /graphify path
|
||||
|
||||
Find the shortest path between two named concepts in the graph. Prefer the CLI when installed:
|
||||
|
||||
```bash
|
||||
graphify path "NODE_A" "NODE_B"
|
||||
```
|
||||
|
||||
If the CLI is unavailable, run it inline:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, sys
|
||||
import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8'))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
a_term = 'NODE_A'
|
||||
b_term = 'NODE_B'
|
||||
|
||||
def find_node(term):
|
||||
term = term.lower()
|
||||
scored = sorted(
|
||||
[(sum(1 for w in term.split() if w in G.nodes[n].get('label','').lower()), n)
|
||||
for n in G.nodes()],
|
||||
reverse=True
|
||||
)
|
||||
return scored[0][1] if scored and scored[0][0] > 0 else None
|
||||
|
||||
src = find_node(a_term)
|
||||
tgt = find_node(b_term)
|
||||
|
||||
if not src or not tgt:
|
||||
print(f'Could not find nodes matching: {a_term!r} or {b_term!r}')
|
||||
sys.exit(0)
|
||||
|
||||
try:
|
||||
path = nx.shortest_path(G, src, tgt)
|
||||
print(f'Shortest path ({len(path)-1} hops):')
|
||||
for i, nid in enumerate(path):
|
||||
label = G.nodes[nid].get('label', nid)
|
||||
if i < len(path) - 1:
|
||||
_raw = G[nid][path[i+1]]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw
|
||||
rel = edge.get('relation', '')
|
||||
conf = edge.get('confidence', '')
|
||||
print(f' {label} --{rel}--> [{conf}]')
|
||||
else:
|
||||
print(f' {label}')
|
||||
except nx.NetworkXNoPath:
|
||||
print(f'No path found between {a_term!r} and {b_term!r}')
|
||||
except nx.NodeNotFound as e:
|
||||
print(f'Node not found: {e}')
|
||||
"
|
||||
```
|
||||
|
||||
Replace `NODE_A` and `NODE_B` with the actual concept names from the user. Then explain the path in plain language - what each hop means, why it's significant.
|
||||
|
||||
After writing the explanation, save it back:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -m graphify save-result --question "Path from NODE_A to NODE_B" --answer "ANSWER" --type path_query --nodes NODE_A NODE_B
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## For /graphify explain
|
||||
|
||||
Give a plain-language explanation of a single node - everything connected to it. Prefer the CLI when installed:
|
||||
|
||||
```bash
|
||||
graphify explain "NODE_NAME"
|
||||
```
|
||||
|
||||
If the CLI is unavailable, run it inline:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, sys
|
||||
import networkx as nx
|
||||
from networkx.readwrite import json_graph
|
||||
from pathlib import Path
|
||||
|
||||
data = json.loads(Path('graphify-out/graph.json').read_text(encoding='utf-8'))
|
||||
G = json_graph.node_link_graph(data, edges='links')
|
||||
|
||||
term = 'NODE_NAME'
|
||||
term_lower = term.lower()
|
||||
|
||||
# Find best matching node
|
||||
scored = sorted(
|
||||
[(sum(1 for w in term_lower.split() if w in G.nodes[n].get('label','').lower()), n)
|
||||
for n in G.nodes()],
|
||||
reverse=True
|
||||
)
|
||||
if not scored or scored[0][0] == 0:
|
||||
print(f'No node matching {term!r}')
|
||||
sys.exit(0)
|
||||
|
||||
nid = scored[0][1]
|
||||
data_n = G.nodes[nid]
|
||||
print(f'NODE: {data_n.get(\"label\", nid)}')
|
||||
print(f' source: {data_n.get(\"source_file\",\"unknown\")}')
|
||||
print(f' type: {data_n.get(\"file_type\",\"unknown\")}')
|
||||
print(f' degree: {G.degree(nid)}')
|
||||
print()
|
||||
print('CONNECTIONS:')
|
||||
for neighbor in G.neighbors(nid):
|
||||
_raw = G[nid][neighbor]; edge = next(iter(_raw.values()), {}) if isinstance(G, nx.MultiGraph) else _raw
|
||||
nlabel = G.nodes[neighbor].get('label', neighbor)
|
||||
rel = edge.get('relation', '')
|
||||
conf = edge.get('confidence', '')
|
||||
src_file = G.nodes[neighbor].get('source_file', '')
|
||||
print(f' --{rel}--> {nlabel} [{conf}] ({src_file})')
|
||||
"
|
||||
```
|
||||
|
||||
Replace `NODE_NAME` with the concept the user asked about. Then write a 3-5 sentence explanation of what this node is, what it connects to, and why those connections are significant. Use the source locations as citations.
|
||||
|
||||
After writing the explanation, save it back:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -m graphify save-result --question "Explain NODE_NAME" --answer "ANSWER" --type explain --nodes NODE_NAME
|
||||
```
|
||||
@@ -0,0 +1,52 @@
|
||||
# graphify reference: transcribe video and audio
|
||||
|
||||
Load this only when `detect` reported one or more `video` files. A corpus with no video never reads this.
|
||||
|
||||
### Step 2.5 - Transcribe video / audio files (only if video files detected)
|
||||
|
||||
Skip this step entirely if `detect` returned zero `video` files.
|
||||
|
||||
Video and audio files cannot be read directly. Transcribe them to text first, then treat the transcripts as doc files in Step 3.
|
||||
|
||||
**Strategy:** Read the god nodes from `graphify-out/.graphify_detect.json` (or the analysis file if it exists from a previous run). You are already a language model — write a one-sentence domain hint yourself from those labels. Then pass it to Whisper as the initial prompt. No separate API call needed.
|
||||
|
||||
**However**, if the corpus has *only* video files and no other docs/code, use the generic fallback prompt: `"Use proper punctuation and paragraph breaks."`
|
||||
|
||||
**Step 1 - Write the Whisper prompt yourself.**
|
||||
|
||||
Read the top god node labels from detect output or analysis, then compose a short domain hint sentence, for example:
|
||||
|
||||
- Labels: `transformer, attention, encoder, decoder` → `"Machine learning research on transformer architectures and attention mechanisms. Use proper punctuation and paragraph breaks."`
|
||||
- Labels: `kubernetes, deployment, pod, helm` → `"DevOps discussion about Kubernetes deployments and Helm charts. Use proper punctuation and paragraph breaks."`
|
||||
|
||||
**Export** it as `GRAPHIFY_WHISPER_PROMPT` (the exact name the transcriber reads — and it must be `export`ed so the child Python process sees it) for the next command.
|
||||
|
||||
**Step 2 - Transcribe:**
|
||||
|
||||
```bash
|
||||
export GRAPHIFY_WHISPER_MODEL=base # or whatever --whisper-model the user passed (must be exported)
|
||||
export GRAPHIFY_WHISPER_PROMPT="<the one-sentence domain hint you composed in Step 1>"
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
from graphify.transcribe import transcribe_all
|
||||
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
video_files = detect.get('files', {}).get('video', [])
|
||||
prompt = os.environ.get('GRAPHIFY_WHISPER_PROMPT', 'Use proper punctuation and paragraph breaks.')
|
||||
|
||||
transcript_paths = transcribe_all(video_files, initial_prompt=prompt)
|
||||
# Write the JSON from Python (NOT a shell '>' redirect): transcribe_all/Whisper
|
||||
# print progress to stdout, which would otherwise corrupt the JSON file (#1392).
|
||||
Path('graphify-out/.graphify_transcripts.json').write_text(json.dumps(transcript_paths, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Transcribed {len(transcript_paths)} file(s)', file=sys.stderr)
|
||||
"
|
||||
```
|
||||
|
||||
After transcription:
|
||||
- Read the transcript paths from `graphify-out/.graphify_transcripts.json`
|
||||
- Add them to the docs list before dispatching semantic subagents in Step 3B
|
||||
- Print how many transcripts were created: `Transcribed N video file(s) -> treating as docs`
|
||||
- If transcription fails for a file, print a warning and continue with the rest
|
||||
|
||||
**Whisper model:** Default is `base`. If the user passed `--whisper-model <name>`, `export GRAPHIFY_WHISPER_MODEL=<name>` (it must be exported, not just assigned) before running the command above.
|
||||
@@ -0,0 +1,221 @@
|
||||
# graphify reference: incremental update and cluster-only
|
||||
|
||||
Load this only when the user passed `--update` or `--cluster-only`. A first-time full build never reads this file.
|
||||
|
||||
## For --update (incremental re-extraction)
|
||||
|
||||
Use when you've added or modified files since the last run. Only re-extracts changed files - saves tokens and time.
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import sys, json
|
||||
from graphify.detect import detect_incremental, save_manifest
|
||||
from pathlib import Path
|
||||
|
||||
from graphify.paths import out_path
|
||||
from graphify.watch import _read_build_excludes, _read_build_gitignore
|
||||
result = detect_incremental(Path('INPUT_PATH'), extra_excludes=_read_build_excludes(out_path()), gitignore=_read_build_gitignore(out_path()))
|
||||
new_total = result.get('new_total', 0)
|
||||
nf = result.get('new_files', {})
|
||||
print(json.dumps({
|
||||
'new_total': new_total,
|
||||
'deleted': len(result.get('deleted_files', [])),
|
||||
'new_by_type': {t: len(v) for t, v in nf.items()},
|
||||
'total_files': result.get('total_files'),
|
||||
'total_words': result.get('total_words'),
|
||||
'skipped_sensitive': result.get('skipped_sensitive', []),
|
||||
}, indent=2, ensure_ascii=False))
|
||||
print('Full result: graphify-out/.graphify_incremental.json')
|
||||
Path('graphify-out/.graphify_incremental.json').write_text(json.dumps(result, ensure_ascii=False), encoding=\"utf-8\")
|
||||
deleted = list(result.get('deleted_files', []))
|
||||
if new_total == 0 and not deleted:
|
||||
print('No files changed since last run. Nothing to update.')
|
||||
raise SystemExit(0)
|
||||
if deleted:
|
||||
print(f'{len(deleted)} deleted file(s) to prune.')
|
||||
if new_total > 0:
|
||||
print(f'{new_total} new/changed file(s) to re-extract.')
|
||||
"
|
||||
```
|
||||
|
||||
Then populate `.graphify_detect.json` so Steps 3A–6 (which read it unconditionally) see the right state for an incremental run. `files` carries the changed subset (drives Step 3A AST + Step 3B0 cache check on only what changed); `all_files` carries the full corpus for any step that needs corpus-wide context:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
r = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\"))
|
||||
Path('graphify-out/.graphify_detect.json').write_text(json.dumps({
|
||||
'files': r.get('new_files', {}),
|
||||
'all_files': r.get('files', {}),
|
||||
'total_files': r.get('new_total', 0),
|
||||
'total_words': r.get('total_words', 0),
|
||||
'skipped_sensitive': r.get('skipped_sensitive', []),
|
||||
'needs_graph': True,
|
||||
}, ensure_ascii=False), encoding=\"utf-8\")
|
||||
"
|
||||
```
|
||||
|
||||
If new files exist, first check whether all changed files are code files:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
result = json.loads(open('graphify-out/.graphify_incremental.json', encoding='utf-8').read()) if Path('graphify-out/.graphify_incremental.json').exists() else {}
|
||||
code_exts = {'.py','.ts','.js','.go','.rs','.java','.cpp','.c','.rb','.swift','.kt','.cs','.scala','.php','.cc','.cxx','.hpp','.h','.kts','.lua','.toc','.f','.F','.f90','.F90','.f95','.F95','.f03','.F03','.f08','.F08'}
|
||||
new_files = result.get('new_files', {})
|
||||
all_changed = [f for files in new_files.values() for f in files]
|
||||
code_only = all(Path(f).suffix.lower() in code_exts for f in all_changed)
|
||||
print('code_only:', code_only)
|
||||
"
|
||||
```
|
||||
|
||||
If `code_only` is True: print `[graphify update] Code-only changes detected - skipping semantic extraction (no LLM needed)`, run Step 3A (AST) on the changed files, then follow only Part B's **Fast path** to overwrite `.graphify_semantic.json` with an empty result and run Part C to create `.graphify_extract.json` from the fresh AST result. Skip semantic cache checks/writes and subagents (Steps B0–B3); do not reuse semantic sidecars from a prior run. Continue to the incremental merge and Steps 4–8 only after Part C has completed.
|
||||
|
||||
If `code_only` is False (any changed file is a doc/paper/image/video): **first, if any changed file is in `new_files['video']`, run `references/transcribe.md` (Step 2.5) on those files, then rewrite `.graphify_detect.json` to move the resulting transcript paths into `files['document']` and drop `files['video']`** — otherwise raw `.mp4/.mp3` paths are fed to semantic subagents as unreadable media (#1392). Then run the full Steps 3A–3C pipeline as normal.
|
||||
|
||||
|
||||
If no new files exist (only deletions), create an empty extraction so the merge step can prune:
|
||||
|
||||
```bash
|
||||
if [ ! -f graphify-out/.graphify_extract.json ]; then
|
||||
echo '[graphify update] Only deletions -- creating empty extraction for merge.'
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps({'nodes':[],'edges':[],'hyperedges':[],'input_tokens':0,'output_tokens':0}), encoding='utf-8')
|
||||
"
|
||||
fi
|
||||
```
|
||||
|
||||
|
||||
Then:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
from graphify.build import build_merge
|
||||
from graphify.detect import save_manifest
|
||||
|
||||
# Load new extraction and incremental state
|
||||
new_extraction = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
incremental = json.loads(Path('graphify-out/.graphify_incremental.json').read_text(encoding=\"utf-8\"))
|
||||
deleted = list(incremental.get('deleted_files', []))
|
||||
# prune_sources is ONLY for genuinely DELETED files. Changed/re-extracted files are
|
||||
# handled by build_merge's replace-on-re-extract (#1344): every source_file in
|
||||
# new_chunks is dropped from the base before merge, so old/stale nodes don't survive.
|
||||
# Do NOT add changed here: with root= passed, prune_set relativizes to the same base
|
||||
# as the freshly merged nodes and would DELETE the re-extracted content (#1178 is moot
|
||||
# now that replace — not the dedup pass — reconciles changed files).
|
||||
prune = list(deleted) or None
|
||||
|
||||
# Use build_merge() — reads graph.json directly without NetworkX round-trip
|
||||
# so edge direction (calls, implements, imports) is always preserved (#801).
|
||||
# Pass root= so prune_sources (absolute paths from detect_incremental) are
|
||||
# relativized to match the graph's relative source_file values; without it
|
||||
# nothing is pruned and stale nodes accumulate on every update (#1361).
|
||||
# directed=IS_DIRECTED: replace IS_DIRECTED with True if --directed was given, else
|
||||
# False. Without it a --directed --update silently rebuilds undirected and collapses
|
||||
# reciprocal A<->B edges (#1392).
|
||||
G = build_merge(
|
||||
[new_extraction],
|
||||
graph_path='graphify-out/graph.json',
|
||||
prune_sources=prune,
|
||||
root='INPUT_PATH',
|
||||
directed=IS_DIRECTED,
|
||||
)
|
||||
print(f'[graphify update] Merged: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges')
|
||||
|
||||
# Write merged result back to .graphify_extract.json so Step 4 sees the full graph
|
||||
merged_out = {
|
||||
'nodes': [{'id': n, **d} for n, d in G.nodes(data=True)],
|
||||
'edges': [
|
||||
# Explicit source/target last so they win over any stale attrs in d.
|
||||
{**{k: val for k, val in d.items() if k not in ('_src', '_tgt', 'source', 'target')},
|
||||
'source': d.get('_src', u), 'target': d.get('_tgt', v)}
|
||||
for u, v, d in G.edges(data=True)
|
||||
],
|
||||
# G.graph["hyperedges"] holds hyperedges from both existing graph.json
|
||||
# and new_extraction (build_merge combines them). Falling back to
|
||||
# new_extraction only would silently drop prior-run hyperedges (#801).
|
||||
'hyperedges': list(G.graph.get('hyperedges', [])),
|
||||
'input_tokens': new_extraction.get('input_tokens', 0),
|
||||
'output_tokens': new_extraction.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_extract.json').write_text(json.dumps(merged_out, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"])} nodes, {len(merged_out[\"edges\"])} edges)')
|
||||
|
||||
# Save manifest so next --update diffs against today's state, not the
|
||||
# prior run's baseline (prevents ghost-node reports on subsequent updates).
|
||||
# root= matches the build_merge call above so the manifest keys stay relative to
|
||||
# the scan root — portable across clones/machines, so --update keeps matching
|
||||
# cached files instead of missing every one after a move (#1417).
|
||||
#
|
||||
# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output
|
||||
# THIS run (new_extraction is this run's fresh extraction, read above before the
|
||||
# merge overwrote the file): a changed doc whose chunk failed must stay unstamped
|
||||
# so the next --update re-queues it, otherwise it is marked done and its content
|
||||
# is lost forever (#2015). Mirrors the library extract path
|
||||
# (cli._stamped_manifest_files + clear_semantic + scan_corpus).
|
||||
from graphify.cli import _stamped_manifest_files
|
||||
_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH'))
|
||||
# Changed semantic files dispatched this run but NOT stamped had their chunk fail
|
||||
# or be omitted; clear any stale semantic_hash so they are re-queued (#1948).
|
||||
_sem_types = ('document', 'paper', 'image')
|
||||
_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl}
|
||||
_stamped = {f for fl in _manifest_files.values() for f in fl}
|
||||
_cleared = _dispatched - _stamped
|
||||
# scan_corpus = the RAW full corpus so in-root files newly excluded since last run
|
||||
# are dropped rather than masquerading as deletions; untouched rows preserved (#1908).
|
||||
_scan = {f for fl in incremental['files'].values() for f in fl}
|
||||
save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None)
|
||||
print('[graphify update] Manifest saved.')
|
||||
"
|
||||
```
|
||||
|
||||
Then run Steps 4–8 on the merged graph as normal.
|
||||
|
||||
After Step 4, show the graph diff:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from graphify.analyze import graph_diff
|
||||
from graphify.build import build_from_json
|
||||
from graphify.paths import load_node_link_graph
|
||||
import networkx as nx
|
||||
from pathlib import Path
|
||||
|
||||
# Load old graph (before update) from backup written before merge
|
||||
old_data = json.loads(Path('graphify-out/.graphify_old.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_old.json').exists() else None
|
||||
new_extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
G_new = build_from_json(new_extract, directed=IS_DIRECTED)
|
||||
|
||||
if old_data:
|
||||
G_old = load_node_link_graph(old_data)
|
||||
diff = graph_diff(G_old, G_new)
|
||||
print(diff['summary'])
|
||||
if diff['new_nodes']:
|
||||
print('New nodes:', ', '.join(n['label'] for n in diff['new_nodes'][:5]))
|
||||
if diff['new_edges']:
|
||||
print('New edges:', len(diff['new_edges']))
|
||||
"
|
||||
```
|
||||
|
||||
Before the merge step, save the old graph: `cp graphify-out/graph.json graphify-out/.graphify_old.json`
|
||||
Clean up after: `rm -f graphify-out/.graphify_old.json`
|
||||
|
||||
---
|
||||
|
||||
## For --cluster-only
|
||||
|
||||
Skip Steps 1–3. Re-run clustering on the existing graph:
|
||||
|
||||
```bash
|
||||
graphify cluster-only .
|
||||
```
|
||||
|
||||
`graphify cluster-only .` is **self-contained**: it re-clusters, names communities, and regenerates `GRAPH_REPORT.md`, `graph.json`, and `graph.html` from the existing graph. **Do not re-run Steps 5–9** — they read intermediate files (`.graphify_extract.json`, `.graphify_detect.json`, `.graphify_analysis.json`) that a prior build's cleanup (Step 9) already deleted, so they raise `FileNotFoundError` (#1392). When it finishes, present the refreshed `GRAPH_REPORT.md` summary as usual.
|
||||
@@ -0,0 +1,8 @@
|
||||
.git
|
||||
**/build
|
||||
managed_components
|
||||
led-driver/managed_components
|
||||
graphify-out
|
||||
sim-controller/data
|
||||
*.elf
|
||||
*.bin
|
||||
@@ -0,0 +1,4 @@
|
||||
# Copy to .env and fill in. Keep .env out of git.
|
||||
DOMAIN=demo.example.com
|
||||
LED_ADMIN_USER=owner
|
||||
LED_ADMIN_PASSWORD=change-me-to-a-long-password
|
||||
+13
@@ -33,3 +33,16 @@ settings.json
|
||||
*.sqlite
|
||||
.pytest_cache/
|
||||
.ropeproject/
|
||||
build/
|
||||
sdkconfig
|
||||
sdkconfig.old
|
||||
managed_components/
|
||||
www_dist/
|
||||
|
||||
# desktop build (make desktop) and its data
|
||||
build-desktop/
|
||||
desktop-data/
|
||||
desktop-data-test/
|
||||
bridge/build-*/
|
||||
sim-driver/build/
|
||||
.env
|
||||
@@ -4,6 +4,3 @@
|
||||
[submodule "led-tool"]
|
||||
path = led-tool
|
||||
url = git@git.technical.kiwi:technicalkiwi/led-tool.git
|
||||
[submodule "led-simulator"]
|
||||
path = led-simulator
|
||||
url = git@git.technical.kiwi:technicalkiwi/led-simulator.git
|
||||
@@ -0,0 +1,15 @@
|
||||
## graphify
|
||||
|
||||
This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships.
|
||||
|
||||
Rules:
|
||||
- When graphify-out/graph.json exists, run `graphify query "<question>"` before your first grep, search or source-file read for a codebase question, and put this rule in every subagent prompt that explores code. Read raw files only for lines the graph or an earlier read already located. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
|
||||
- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing.
|
||||
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context.
|
||||
- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost).
|
||||
- A committed graphify-out/graph.json is rewritten whole on every build, so a large diff to it in a pull request is expected. Do not flag it as churn or suggest skipping the graph rebuild or its hooks.
|
||||
|
||||
## ESP-IDF
|
||||
|
||||
- Build ESP-IDF firmware with ESP-IDF v6.1 only: the `espressif/idf:v6.1` Docker image (the Makefile's `IDF_IMAGE`), or the same v6.1 install. Do not build with the host install at `~/esp/esp-idf` (v5.5.1), and do not use the v5.5.5 image.
|
||||
- Build in a fresh build directory created by the v6.1 image. A `build/` folder made outside it fails CMake with a cache mismatch, so delete it first if it is host-made.
|
||||
@@ -0,0 +1,3 @@
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
project(led_controller)
|
||||
@@ -0,0 +1,28 @@
|
||||
# Production image of the controller demo: ESP-IDF's linux target running the simulated rig. No hardware.
|
||||
# The web UI is embedded in the binary (main/CMakeLists.txt packs www_dist/www.pack), so no Node is needed.
|
||||
# Build from the repo root: docker build -f Dockerfile.prod -t led-controller:prod .
|
||||
# Run with the compose file: docker compose -f docker-compose.prod.yml up -d (see .env.prod.example)
|
||||
|
||||
FROM espressif/idf:v6.1 AS build
|
||||
WORKDIR /project
|
||||
COPY . .
|
||||
RUN bash -c ". /opt/esp/idf/export.sh && cd sim-controller && idf.py build"
|
||||
|
||||
FROM debian:trixie-slim
|
||||
# The two shared libraries the binary needs, taken from the build image
|
||||
COPY --from=build /usr/lib/x86_64-linux-gnu/libbsd.so.0.12.1 /usr/lib/x86_64-linux-gnu/libbsd.so.0
|
||||
COPY --from=build /usr/lib/x86_64-linux-gnu/libmd.so.0.1.0 /usr/lib/x86_64-linux-gnu/libmd.so.0
|
||||
COPY --from=build /project/sim-controller/build/led_controller_sim.elf /usr/local/bin/led_controller_sim
|
||||
RUN useradd --system --uid 10001 --no-create-home --shell /usr/sbin/nologin led \
|
||||
&& mkdir /data && chown led:led /data
|
||||
ENV LED_SIM_DB=/data \
|
||||
LED_SIM_HTTP_PORT=8080 \
|
||||
LED_SIM_BRIDGE=127.0.0.1:8766
|
||||
USER led
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 8080
|
||||
# Bash's /dev/tcp needs no extra packages: healthy when /status answers 200
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD bash -c 'exec 3<>/dev/tcp/127.0.0.1/8080 && printf "GET /status HTTP/1.0\r\n\r\n" >&3 && head -n1 <&3 | grep -q " 200"'
|
||||
STOPSIGNAL SIGTERM
|
||||
ENTRYPOINT ["/usr/local/bin/led_controller_sim"]
|
||||
@@ -0,0 +1,26 @@
|
||||
# Desktop image of the controller firmware built for ESP-IDF's linux target: no hardware, no LAN.
|
||||
# The web UI is embedded in the binary (main/CMakeLists.txt packs www_dist/www.pack), so no Node is needed.
|
||||
# Build from the repo root: docker build -f Dockerfile.sim -t led-controller-sim .
|
||||
# Run: docker run --rm -p 8080:8080 -v led-sim-data:/data led-controller-sim
|
||||
# Editing needs a sign-in. Set the first admin with LED_ADMIN_USER and LED_ADMIN_PASSWORD (8+ characters);
|
||||
# until a user exists the controller is open to everyone, so set them on the first run.
|
||||
# docker run ... -e LED_ADMIN_USER=owner -e LED_ADMIN_PASSWORD='change-me-now' led-controller-sim
|
||||
|
||||
FROM espressif/idf:v6.1 AS build
|
||||
WORKDIR /project
|
||||
COPY . .
|
||||
RUN bash -c ". /opt/esp/idf/export.sh && cd sim-controller && idf.py build"
|
||||
|
||||
FROM debian:trixie
|
||||
# The two shared libraries the binary needs, taken from the build image (no package install needed)
|
||||
COPY --from=build /usr/lib/x86_64-linux-gnu/libbsd.so.0.12.1 /usr/lib/x86_64-linux-gnu/libbsd.so.0
|
||||
COPY --from=build /usr/lib/x86_64-linux-gnu/libmd.so.0.1.0 /usr/lib/x86_64-linux-gnu/libmd.so.0
|
||||
COPY --from=build /project/sim-controller/build/led_controller_sim.elf /usr/local/bin/led_controller_sim
|
||||
RUN useradd --system --uid 10001 --no-create-home led && mkdir /data && chown led /data
|
||||
ENV LED_SIM_DB=/data \
|
||||
LED_SIM_HTTP_PORT=8080 \
|
||||
LED_SIM_BRIDGE=127.0.0.1:8766
|
||||
USER led
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 8080
|
||||
ENTRYPOINT ["/usr/local/bin/led_controller_sim"]
|
||||
@@ -0,0 +1,229 @@
|
||||
# Build, flash and monitor the ESP32-S3 controller (ESP-IDF in Docker).
|
||||
#
|
||||
# make build, flash, then monitor (same as: make all)
|
||||
# make help list the targets
|
||||
# make build minify the web UI, then build the firmware
|
||||
# make flash flash the last build to the S3 (does not build)
|
||||
# make monitor serial console (Ctrl-] to quit)
|
||||
# make flash-monitor
|
||||
#
|
||||
# Override on the command line, e.g. make flash PORT=/dev/ttyACM1
|
||||
|
||||
PORT ?= /dev/ttyACM0
|
||||
IDF_IMAGE ?= espressif/idf:v6.1
|
||||
SECS ?= 10
|
||||
|
||||
DOCKER_RUN = docker run --rm -v "$(CURDIR)":/project -w /project
|
||||
DEVICE = --device=$(PORT)
|
||||
# Interactive terminal only when there is one (monitor needs it, CI does not have it)
|
||||
TTY := $(shell [ -t 0 ] && echo -it)
|
||||
|
||||
# Rootless Docker maps container root to you, so files must end up owned by uid 0 inside; with a
|
||||
# normal daemon they must be owned by you.
|
||||
OWNER := $(shell docker info --format '{{.SecurityOptions}}' 2>/dev/null | grep -q rootless && echo 0:0 || echo $$(id -u):$$(id -g))
|
||||
FIX_OWNER = chown -R $(OWNER) /project
|
||||
|
||||
.PHONY: all help web build flash ota monitor flash-monitor log size menuconfig clean fullclean erase test test-driver test-envelope test-api test-browser sim-build sim sim-wasm sim-pack sim-web test-sim
|
||||
|
||||
# Default target: build, flash, then open the console (run in order, so not with make -j)
|
||||
all: build flash monitor
|
||||
|
||||
help:
|
||||
@echo "LED controller (ESP32-S3):"
|
||||
@echo " make [all] build, flash, then open the console"
|
||||
@echo " make build minify + gzip the web UI, then build the firmware"
|
||||
@echo " make flash flash the last build (firmware and web UI) to PORT ($(PORT)); does not build"
|
||||
@echo " make ota [HOST=ip] build, then upload the firmware (web UI included) over Ethernet (no USB);"
|
||||
@echo " HOST defaults to led-controller.local"
|
||||
@echo " make monitor serial console on PORT (Ctrl-] to quit)"
|
||||
@echo " make flash-monitor flash the last build, then open the console"
|
||||
@echo " make log [SECS=10] print the console for SECS seconds and exit (no terminal needed)"
|
||||
@echo " make web only rebuild the web UI (needs node/npx)"
|
||||
@echo " make size memory use of the last build"
|
||||
@echo " make menuconfig change the ESP-NOW channel, marker pin, W5500 pins, ..."
|
||||
@echo " make erase erase the whole flash (loses the stored data)"
|
||||
@echo " make clean remove the build directory (keeps sdkconfig)"
|
||||
@echo " make fullclean also remove sdkconfig and the web UI output"
|
||||
@echo " make test driver unit tests + C/Python envelope cross-check"
|
||||
@echo " make test-api HOST=ip HTTP API tests against a running controller (data restored afterwards)"
|
||||
@echo " make test-browser HOST=ip Selenium UI tests against a running controller (needs selenium + chromedriver)"
|
||||
@echo "Simulated rig (no hardware): the controller built for linux + led-simulator's 16 drivers over a UDP bridge"
|
||||
@echo " make sim-build build the controller for the host (sim-controller/build/led_controller_sim.elf)"
|
||||
@echo " make sim build, then run the headless rig and the controller (http://localhost:8080/)"
|
||||
@echo " make sim-wasm build the browser drivers: led-driver (C++) to WebAssembly, with Emscripten in Docker"
|
||||
@echo " make sim-web build all of it and run one program: the controller on http://localhost:8080/ with the"
|
||||
@echo " simulated rig (rooms, every bar a driver) on /sim/ and the pattern gallery on /sim/gallery"
|
||||
@echo " make test-sim rig tests (groups, field patterns, sequences) + the API tests against make sim"
|
||||
@echo "variables: PORT=$(PORT) IDF_IMAGE=$(IDF_IMAGE)"
|
||||
|
||||
# --- web UI ---------------------------------------------------------------
|
||||
web:
|
||||
tools/build_web.sh
|
||||
|
||||
# --- firmware -------------------------------------------------------------
|
||||
# The build time in the image comes from esp_app_desc.c, which nothing else recompiles (a web-only
|
||||
# change just relinks), so drop its object and every build gets the current time.
|
||||
APP_DESC_OBJ = build/esp-idf/esp_app_format/CMakeFiles/__idf_esp_app_format.dir/esp_app_desc.c.obj
|
||||
|
||||
build: web
|
||||
$(DOCKER_RUN) $(IDF_IMAGE) bash -c 'rm -f $(APP_DESC_OBJ); idf.py build; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
flash:
|
||||
$(DOCKER_RUN) $(DEVICE) $(IDF_IMAGE) bash -c 'idf.py -p $(PORT) flash; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
ota: HOST ?= led-controller.local
|
||||
ota: build
|
||||
python3 tools/ota.py $(HOST)
|
||||
|
||||
monitor:
|
||||
$(DOCKER_RUN) $(TTY) $(DEVICE) $(IDF_IMAGE) idf.py -p $(PORT) monitor
|
||||
|
||||
flash-monitor: flash monitor
|
||||
|
||||
# Non-interactive: reset the board and print what it says for SECS seconds
|
||||
log:
|
||||
python3 tools/serial_log.py $(PORT) $(SECS)
|
||||
|
||||
size:
|
||||
$(DOCKER_RUN) $(IDF_IMAGE) bash -c 'idf.py size; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
menuconfig:
|
||||
$(DOCKER_RUN) -it $(IDF_IMAGE) bash -c 'idf.py menuconfig; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
erase:
|
||||
$(DOCKER_RUN) $(DEVICE) $(IDF_IMAGE) idf.py -p $(PORT) erase-flash
|
||||
|
||||
clean:
|
||||
rm -rf build
|
||||
|
||||
fullclean: clean
|
||||
rm -rf sdkconfig sdkconfig.old www_dist
|
||||
|
||||
# --- tests ----------------------------------------------------------------
|
||||
test: test-driver test-envelope
|
||||
|
||||
# Host-side unit tests for the driver's link layer and beat sync (pure Python)
|
||||
test-driver:
|
||||
python3 tools/run_driver_tests.py
|
||||
|
||||
# HTTP API tests (ported from v1) against a running controller; v1 features v2 lacks are xfail
|
||||
test-api:
|
||||
@test -n "$(HOST)" || { echo "usage: make test-api HOST=<controller ip>"; exit 1; }
|
||||
python3 -m pytest test/api -q --host $(HOST)
|
||||
|
||||
# Selenium UI tests (ported from v1); they change the controller's data and do not restore it
|
||||
test-browser:
|
||||
@test -n "$(HOST)" || { echo "usage: make test-browser HOST=<controller ip>"; exit 1; }
|
||||
LED_CONTROLLER_RUN_BROWSER_TESTS=1 LED_HOST=$(HOST) python3 -m pytest test/browser -q
|
||||
|
||||
# The C envelope packer must produce the same bytes as test/binary_envelope.py
|
||||
test-envelope:
|
||||
@tmp=$$(mktemp -d); \
|
||||
(cd test && python3 env_check.py) > $$tmp/py.txt; \
|
||||
docker run --rm -v "$(CURDIR)":/project -w /project/test $(IDF_IMAGE) bash -c \
|
||||
'gcc -Wall -I/opt/esp/idf/components/json/cJSON env_host.c ../main/envelope.c /opt/esp/idf/components/json/cJSON/cJSON.c -o /tmp/t -lm && /tmp/t' 2>&1 | tail -4 | head -3 > $$tmp/c.txt; \
|
||||
if diff -q $$tmp/py.txt $$tmp/c.txt >/dev/null; then echo "envelope: C and Python agree"; rm -rf $$tmp; \
|
||||
else echo "envelope: MISMATCH"; diff $$tmp/py.txt $$tmp/c.txt; rm -rf $$tmp; exit 1; fi
|
||||
|
||||
# --- simulated rig --------------------------------------------------------
|
||||
# The firmware built for ESP-IDF's linux target runs on this computer and serves the simulated rig itself at
|
||||
# /sim/ (sim-controller/components/sim/sim_rig.c): every bar is led-driver's C++ firmware built to WebAssembly
|
||||
# (sim-driver/, `make sim-wasm`), laid out in a room, and gets the controller's ESP-NOW frames over the page's
|
||||
# WebSocket (the "browser" bridge). bridge_sim.py is the older headless rig over UDP, whose drivers ran the
|
||||
# MicroPython led-driver/src (since removed).
|
||||
SIM_DB ?= $(CURDIR)/sim-controller/data
|
||||
SIM_RUN = mkdir -p $(SIM_DB) && LED_SIM_DB=$(SIM_DB) LED_SIM_BRIDGE=127.0.0.1:8766 sim-controller/build/led_controller_sim.elf
|
||||
|
||||
# The rig's page and drivers for the host builds to embed (www_dist/sim.pack; empty without `make sim-wasm`)
|
||||
sim-pack:
|
||||
tools/build_sim_web.sh
|
||||
|
||||
sim-build: web sim-pack
|
||||
$(DOCKER_RUN) $(IDF_IMAGE) bash -c 'cd sim-controller && idf.py build; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
sim: sim-build
|
||||
@trap 'kill 0' INT TERM EXIT; python3 led-simulator/bridge_sim.py & $(SIM_RUN)
|
||||
|
||||
# Emscripten in Docker (sim-driver/build.sh): led_driver (one strip) and led_fx (the pattern gallery)
|
||||
EMSDK_IMAGE ?= emscripten/emsdk:4.0.12
|
||||
sim-wasm:
|
||||
EMSDK_IMAGE=$(EMSDK_IMAGE) sim-driver/build.sh
|
||||
|
||||
sim-web: sim-wasm sim-build
|
||||
mkdir -p $(SIM_DB) && LED_SIM_DB=$(SIM_DB) LED_SIM_BRIDGE=browser sim-controller/build/led_controller_sim.elf
|
||||
|
||||
test-sim:
|
||||
python3 -m pytest test/api -q --host localhost:8080
|
||||
|
||||
# --- desktop image (no hardware): the sim controller in Docker, web UI on localhost:8080 ----
|
||||
# DOCKER_PULL=--pull=false (default) uses the local espressif/idf:v6.1 and debian:trixie images; set DOCKER_PULL= to refresh them
|
||||
DOCKER_PULL ?= --pull=false
|
||||
docker-sim:
|
||||
DOCKER_BUILDKIT=0 docker build $(DOCKER_PULL) -f Dockerfile.sim -t led-controller-sim .
|
||||
|
||||
docker-sim-run:
|
||||
docker run --rm -p 8080:8080 -v led-sim-data:/data led-controller-sim
|
||||
|
||||
# --- USB ESP-NOW bridge board (bridge/): ESP-NOW for a computer with no radio ------------------
|
||||
# Two chips: ESP32-S3 (bridge-*-s3, with the I2S microphone) and ESP32-C3 (bridge-*-c3, no mic). Each has its own
|
||||
# build dir (bridge/build-s3, bridge/build-c3). Flash with BRIDGE=<port>, e.g. make bridge-flash-s3 BRIDGE=/dev/ttyACM2.
|
||||
# BRIDGE=/dev/ttyACM1 is the default for `make rig`, which runs the sim-controller build against it (no simulator).
|
||||
BRIDGE ?= /dev/ttyACM1
|
||||
define BRIDGE_BUILD
|
||||
$(DOCKER_RUN) $(IDF_IMAGE) bash -c 'cd bridge; if [ ! -f build-$(1)/CMakeCache.txt ]; then idf.py -B build-$(1) -D SDKCONFIG=build-$(1)/sdkconfig set-target $(2); fi; idf.py -B build-$(1) -D SDKCONFIG=build-$(1)/sdkconfig build; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
endef
|
||||
bridge-build-s3:
|
||||
$(call BRIDGE_BUILD,s3,esp32s3)
|
||||
bridge-build-c3:
|
||||
$(call BRIDGE_BUILD,c3,esp32c3)
|
||||
bridge-build: bridge-build-s3 bridge-build-c3
|
||||
|
||||
bridge-flash-s3: bridge-build-s3
|
||||
$(DOCKER_RUN) --device=$(BRIDGE) $(IDF_IMAGE) bash -c 'cd bridge && idf.py -B build-s3 -D SDKCONFIG=build-s3/sdkconfig -p $(BRIDGE) flash; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
bridge-flash-c3: bridge-build-c3
|
||||
$(DOCKER_RUN) --device=$(BRIDGE) $(IDF_IMAGE) bash -c 'cd bridge && idf.py -B build-c3 -D SDKCONFIG=build-c3/sdkconfig -p $(BRIDGE) flash; rc=$$?; $(FIX_OWNER); exit $$rc'
|
||||
|
||||
rig: sim-build
|
||||
@test -n "$(BRIDGE)" || { echo "usage: make rig BRIDGE=/dev/ttyACM1"; exit 1; }
|
||||
mkdir -p $(SIM_DB) && LED_SIM_DB=$(SIM_DB) LED_SIM_BRIDGE=$(BRIDGE) sim-controller/build/led_controller_sim.elf
|
||||
|
||||
# --- desktop: led-controller as a program, no ESP-IDF (desktop/); make deb for a headless install ---
|
||||
DESKTOP_BUILD ?= build-desktop
|
||||
DESKTOP_DATA ?= $(CURDIR)/desktop-data
|
||||
DESKTOP_PORT ?= 8080
|
||||
|
||||
# Rebuilt when any web source changes
|
||||
WEB_SRC := $(wildcard web/static/* web/templates/* web/*.html) tools/build_web.sh
|
||||
www_dist/www.pack: $(WEB_SRC)
|
||||
tools/build_web.sh
|
||||
|
||||
desktop: www_dist/www.pack sim-pack
|
||||
cmake -S desktop -B $(DESKTOP_BUILD) -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build $(DESKTOP_BUILD) -j$(shell nproc)
|
||||
|
||||
# The web UI is on http://localhost:$(DESKTOP_PORT)/; the bridges are set there (Settings > Network > Bridges), not here
|
||||
desktop-run: desktop
|
||||
$(DESKTOP_BUILD)/led-controller --db $(DESKTOP_DATA) --port $(DESKTOP_PORT)
|
||||
|
||||
test-desktop: desktop
|
||||
mkdir -p $(DESKTOP_DATA)-test
|
||||
$(DESKTOP_BUILD)/led-controller --db $(DESKTOP_DATA)-test --port 8081 > $(DESKTOP_DATA)-test/server.log 2>&1 & pid=$$!; sleep 2; python3 -m pytest test/api -q --host localhost:8081 --ignore=test/api/test_rig_sim.py; rc=$$?; kill $$pid 2>/dev/null; wait $$pid 2>/dev/null; exit $$rc
|
||||
|
||||
# led-cli (led-tool/) and the driver firmware image (led-driver/, built by its ESP-IDF v6.1 image) go into the .deb
|
||||
LED_CLI_BIN := led-tool/dist/led-cli
|
||||
LED_DRIVER_BIN := led-driver/build-v61/led_driver.bin
|
||||
LED_TOOL_SRC := $(wildcard led-tool/*.py led-tool/lib/*.py)
|
||||
|
||||
led-cli: $(LED_CLI_BIN)
|
||||
|
||||
$(LED_CLI_BIN): $(LED_TOOL_SRC) led-tool/led-cli.spec
|
||||
cd led-tool && pipenv run pyinstaller --clean --onefile --name led-cli --paths lib cli.py
|
||||
|
||||
deb: www_dist/www.pack $(LED_CLI_BIN)
|
||||
@test -f $(LED_DRIVER_BIN) || { echo "missing $(LED_DRIVER_BIN): build led-driver with the espressif/idf:v6.1 image first"; exit 1; }
|
||||
cmake -S desktop -B $(DESKTOP_BUILD) -DCMAKE_BUILD_TYPE=Release -DLED_BUNDLE_TOOLS=ON
|
||||
cmake --build $(DESKTOP_BUILD) -j$(shell nproc)
|
||||
cd $(DESKTOP_BUILD) && cpack -G DEB
|
||||
|
||||
.PHONY: desktop desktop-run test-desktop deb led-cli bridge-build bridge-build-s3 bridge-build-c3 bridge-flash-s3 bridge-flash-c3 rig
|
||||
@@ -1,30 +0,0 @@
|
||||
[[source]]
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
mpremote = "*"
|
||||
pyserial = "*"
|
||||
esptool = "*"
|
||||
pyjwt = "*"
|
||||
watchfiles = "*"
|
||||
requests = "*"
|
||||
selenium = "*"
|
||||
adafruit-ampy = "*"
|
||||
microdot = "*"
|
||||
websockets = "*"
|
||||
|
||||
[dev-packages]
|
||||
pytest = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.11"
|
||||
|
||||
[scripts]
|
||||
web = "python /home/pi/led-controller/tests/web.py"
|
||||
watch = "python -m watchfiles 'python tests/web.py' src tests"
|
||||
install = "pipenv install"
|
||||
run = "sh -c 'cd src && python main.py'"
|
||||
dev = "watchfiles \"sh -c 'cd src && python main.py'\" src"
|
||||
help-pdf = "sh scripts/build_help_pdf.sh"
|
||||
Generated
-1026
File diff suppressed because it is too large.
Load diff
@@ -1,61 +1,54 @@
|
||||
# led-controller
|
||||
|
||||
LED controller web app for managing profiles, **zones**, presets, and colour palettes, and sending commands to LED devices. Outbound paths include:
|
||||
ESP32-S3 controller for LED drivers: W5500 Ethernet, web UI, I2S microphone beat detection and an ESP-NOW link to the drivers. The firmware is C (ESP-IDF) at the repository root; see [`docs/BENCH_TEST.md`](docs/BENCH_TEST.md) for the bench tests.
|
||||
|
||||
- **Serial → ESP-NOW bridge**: JSON lines over UART to an ESP32 that forwards ESP-NOW frames (configure `serial_port` and baud in `settings.json` / Settings model).
|
||||
- **Wi-Fi LED drivers**: TCP JSON lines (default port **8765** on the Pi; drivers discover the controller via **UDP 8766** broadcast).
|
||||
It keeps profiles, zones, presets, device groups, sequences, palettes, patterns and outputs in its flash, and serves the web UI and JSON API ([`docs/API.md`](docs/API.md)). Zones play to device groups. Sequences advance on the beats the microphone detects, or on a BPM clock while audio is stopped. Profiles, presets and sequences import and export as JSON bundles.
|
||||
|
||||
## Run
|
||||
|
||||
- One-time setup for port 80 without root: `sudo scripts/setup-port80.sh`
|
||||
- Start app: `pipenv run run` (override listen port with the **`PORT`** environment variable)
|
||||
- Dev watcher (auto-restart on `src/` changes): `pipenv run dev`
|
||||
- Regenerate **`docs/help.pdf`** from **`docs/help.md`**: `pipenv run help-pdf` (requires **pandoc** and **chromium** on the host)
|
||||
|
||||
## UI modes
|
||||
|
||||
- **Run mode**: focused control view. Select zones/presets and apply profiles. Editing actions are hidden.
|
||||
- **Edit mode**: management view. Shows **Zones**, Presets, Patterns, Colour Palette, and Send Presets controls, plus per-tile preset edit/remove and drag-reorder.
|
||||
|
||||
## Profiles
|
||||
|
||||
- Applying a profile updates session scope and refreshes the active zone content.
|
||||
- In **Run mode**, Profiles supports apply-only behaviour (no create/clone/delete).
|
||||
- In **Edit mode**, Profiles supports create/clone/delete.
|
||||
- Creating a profile always creates a populated `default` zone (starter presets).
|
||||
- Optional **DJ zone** seeding creates:
|
||||
- `dj` zone bound to device name `dj`
|
||||
- starter DJ presets (rainbow, single colour, transition)
|
||||
|
||||
## Preset colours and palette linking
|
||||
|
||||
- In preset editor, selecting a colour picker value auto-adds it when the picker closes.
|
||||
- Use **From Palette** to add a palette-linked preset colour.
|
||||
- Linked colours are stored as palette references and shown with a `P` badge.
|
||||
- When profile palette colours change, linked preset colours update across that profile.
|
||||
|
||||
## API docs
|
||||
|
||||
- Main API reference: `docs/API.md`
|
||||
|
||||
## Driver pattern modules
|
||||
|
||||
Pattern **`.py`** sources live under **`led-driver/src/patterns`**. The Pi app resolves that path via `util.driver_patterns.driver_patterns_dir()`. If you deploy without that tree next to the app, set **`LED_CONTROLLER_PATTERNS_DIR`** to the directory that contains those files.
|
||||
|
||||
## Animation engine (Pi-side)
|
||||
|
||||
`src/engine/` lets the Pi drive LEDs itself, in addition to the ESP32 drivers:
|
||||
|
||||
- **Pixel generator**: runs the **unchanged** `led-driver/src/patterns/*.py` under CPython (a `utime` shim plus a NeoPixel-like buffer) with the same preset JSON the drivers get, and streams each frame to **pixel outputs**: DDP, Art-Net, sACN (E1.31), or USB-DMX (Enttec Pro / Open DMX). Configure outputs in **Outputs** (edit mode) and add an output's name to a zone's devices.
|
||||
- **Preset sequencer**: **Sequences** play timed chains of presets on a zone, switching both the pixel outputs and the zone's ESP32 drivers. Sequences resume after a restart.
|
||||
- ESP32 firmware is unchanged; drivers still get presets + `select`.
|
||||
|
||||
CLI (run from `src/`):
|
||||
## Build and flash
|
||||
|
||||
```bash
|
||||
python -m engine.cli --output "ddp://192.168.1.60?pixels=300" --test
|
||||
python -m engine.cli --output "artnet://192.168.1.61?universe=0&pixels=170" --preset 16
|
||||
python -m engine.cli --pattern rainbow --n1 2 --leds 60 # terminal preview
|
||||
make build # minify the web UI from web/, then build the firmware (ESP-IDF in Docker)
|
||||
make flash # flash over USB
|
||||
make ota [HOST=<ip>] # update firmware (web UI included) over Ethernet, no USB (POST /ota); HOST defaults to led-controller.local
|
||||
make monitor # serial console
|
||||
make test # driver unit tests and the C/Python envelope cross-check
|
||||
make test-api HOST=<ip> # HTTP API tests against a running controller (its data is restored afterwards)
|
||||
make test-browser HOST=<ip> # Selenium UI tests (needs selenium and chromedriver)
|
||||
```
|
||||
|
||||
Tests: `pipenv run pytest tests/engine_tests`.
|
||||
`make` lists every target. The web UI sources are in `web/` (`static/`, `templates/`, `audio.html`); `tools/build_web.sh` also packs the led-driver pattern files into the firmware, so the controller can send them to drivers.
|
||||
|
||||
## Desktop
|
||||
|
||||
The same controller runs as a program on Linux, with no ESP-IDF: `make desktop-run` (then add the bridge under Settings > Network > Bridges), or a `.deb` with a systemd service (`make deb`). See [`desktop/README.md`](desktop/README.md). The USB ESP-NOW bridge board that gives a computer its radio is in [`bridge/`](bridge/README.md).
|
||||
|
||||
## Other directories
|
||||
|
||||
- `led-driver/` (submodule): MicroPython firmware for the ESP32 LED drivers.
|
||||
- `led-tool/` (submodule): `led-cli`, configures drivers over serial and updates the controller over Ethernet (`led-cli --ota HOST`).
|
||||
- `led-simulator/` (submodule), `espnow-sender/`: test helpers.
|
||||
- `db/`: the records a controller starts with (the `default` profile and group the firmware seeds at first boot, plus pattern metadata), importable with `PUT /db/<name>`.
|
||||
- `test/api/`: the API tests (`test/api/README.md`); `test/browser/`: the UI tests.
|
||||
|
||||
## Docs
|
||||
|
||||
- API reference: `docs/API.md`
|
||||
- User guide: `docs/help.md` (`scripts/build_help_pdf.sh` regenerates `docs/help.pdf`)
|
||||
- Specification: `docs/SPECIFICATION.md`
|
||||
|
||||
## Network
|
||||
|
||||
The controller advertises `<hostname>.local` over mDNS (default `led-controller`, plus an `_http._tcp` service). Both settings below come from `/settings` and apply on reboot.
|
||||
|
||||
`static_ips` is a list of networks to try, in order, each time the Ethernet link comes up. The first entry whose gateway answers ARP is used; if none does (or the list is empty) the controller uses DHCP. `netmask` defaults to `255.255.255.0`, `dns` to the gateway.
|
||||
|
||||
If DHCP gets no answer within 15 seconds, the controller takes `192.168.4.1/24`. For a cable straight from a PC, give the PC a static IP on that subnet, such as `192.168.4.2/24`, and open `http://192.168.4.1/`. The controller tries DHCP again only when the link next comes up.
|
||||
|
||||
```bash
|
||||
curl -X PUT http://led-controller.local/settings/settings -H 'Content-Type: application/json' -d '{
|
||||
"hostname": "stage",
|
||||
"static_ips": [
|
||||
{"ip": "192.168.1.50", "netmask": "255.255.255.0", "gw": "192.168.1.1"},
|
||||
{"ip": "10.0.0.50", "netmask": "255.255.255.0", "gw": "10.0.0.1"}
|
||||
]}'
|
||||
```
|
||||
@@ -0,0 +1,4 @@
|
||||
# USB ESP-NOW bridge: lets a computer run led-controller against ESP-NOW drivers. See README.md.
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
include($ENV{IDF_PATH}/tools/cmake/project.cmake)
|
||||
project(led_bridge)
|
||||
@@ -0,0 +1,80 @@
|
||||
# USB ESP-NOW bridge
|
||||
|
||||
An ESP32 board that gives a computer an ESP-NOW radio. Two builds: an ESP32-S3 (the one with an I2S
|
||||
microphone, below) and an ESP32-C3 (ESP-NOW only). `led-controller` running on the computer
|
||||
sends its ESP-NOW frames to the board over USB, and the board puts them on the air; frames from the
|
||||
drivers come back the same way.
|
||||
|
||||
```
|
||||
led-controller (host) --SLIP over USB-Serial-JTAG--> bridge board --ESP-NOW--> LED drivers
|
||||
<------------------------------- <--------
|
||||
```
|
||||
|
||||
## Build and flash
|
||||
|
||||
Built with ESP-IDF v6.1 (`espressif/idf:v6.1`), one build directory per chip (`bridge/build-s3`,
|
||||
`bridge/build-c3`):
|
||||
|
||||
```bash
|
||||
make bridge-build-s3 # or bridge-build-c3, or bridge-build for both
|
||||
make bridge-flash-s3 BRIDGE=/dev/ttyACM2 # overwrites the board's firmware
|
||||
make bridge-flash-c3 BRIDGE=/dev/ttyACM1
|
||||
```
|
||||
|
||||
Flash only the board you mean: the flash targets overwrite whatever is on the port.
|
||||
|
||||
## Running the controller against it
|
||||
|
||||
```bash
|
||||
make rig BRIDGE=/dev/ttyACM1 # the sim-controller build, web UI on http://localhost:8080/
|
||||
```
|
||||
|
||||
Or set the bridge in settings (`"bridge": "/dev/ttyACM1"`), or `LED_SIM_BRIDGE=/dev/ttyACM1` for the sim.
|
||||
The controller reconnects if the board is unplugged. Its status (`GET /status`, field `bridge_links`, one entry per bridge) shows
|
||||
whether the board is linked, its MAC and channel, and the counters.
|
||||
|
||||
## Microphone (S3)
|
||||
|
||||
The S3 build reads an I2S microphone (an INMP441 wired as on the controller's XIAO ESP32-S3:
|
||||
SCK GPIO 5, WS GPIO 6, SD GPIO 43, configurable under "Bridge microphone (I2S)" in menuconfig). While the
|
||||
desktop records from it, the board sends 10 ms blocks of 16 kHz mono audio (AUDIO frames, below).
|
||||
The desktop lists it in the audio input list as "I2S microphone on the bridge board". The C3 build has no
|
||||
microphone and reports that in INFO.
|
||||
|
||||
## Channel
|
||||
|
||||
ESP-NOW runs on one Wi-Fi channel, and the drivers and the bridge must share it. The bridge starts on
|
||||
channel 6. Set the channel the drivers use (the access point's, or the router's when a driver is joined to
|
||||
one), either by a SET_CHANNEL frame (`bridge_serial_set_channel`) or by changing it on the board's NVS.
|
||||
|
||||
## Protocol
|
||||
|
||||
SLIP frames, each `[type][body][crc16 LE]` (CRC-16/CCITT-FALSE over type and body). The single source is
|
||||
[`main/bridge_proto.h`](main/bridge_proto.h), shared by the firmware and the host
|
||||
(`main/bridge_serial.c`).
|
||||
|
||||
| type | direction | body |
|
||||
| --- | --- | --- |
|
||||
| `00` PKT | host → board | `dst_mac(6) + packet` → ESP-NOW |
|
||||
| `00` PKT | board → host | `src_mac(6) + packet` for every ESP-NOW frame received |
|
||||
| `10` HELLO | host → board | (empty); answered with INFO |
|
||||
| `11` INFO | board → host | version, MAC, channel, tx, tx_fail, rx, drop counters |
|
||||
| `12` SET_CHANNEL | host → board | channel (1-13); answered with INFO |
|
||||
| `13` SET_MIC | host → board | on (1) or off (0): send microphone audio |
|
||||
| `40` AUDIO | board → host | 160 16-bit little-endian mono samples at 16 kHz, while the mic is on |
|
||||
| `30` LOG | board → host | a log line |
|
||||
|
||||
INFO's last byte is capabilities: bit 0 is set when the board has a microphone.
|
||||
|
||||
The board sends one PKT at a time and waits for each send's result, so a burst cannot overrun the radio.
|
||||
|
||||
## Ethernet (W5500)
|
||||
|
||||
The S3 board also has a W5500 on SPI (XIAO pins by default: SCLK 7, MISO 8, MOSI 9, CS 2, INT 3, RST 1; change them in
|
||||
`make menuconfig` under "Bridge Ethernet (W5500)"). It takes DHCP and answers UDP on port 8766, with the same datagrams as
|
||||
the simulator's bridge: send `dst_mac + packet` and the board puts it on ESP-NOW; each frame it hears comes back to the last
|
||||
sender as `src_mac + packet`.
|
||||
|
||||
The board's address is in INFO (bytes 25-28, with the Ethernet-up bit in byte 24), so the desktop shows it under the board's
|
||||
status (`ip`). Add it as a network bridge with `<ip>:8766`. Don't use the USB and the UDP path for the same board at once:
|
||||
each frame would go out over the air twice.
|
||||
@@ -0,0 +1,34 @@
|
||||
dependencies:
|
||||
espressif/w5500:
|
||||
component_hash: f76a95ce7a6029cf93aa39d47067588b24263852031f11c71e5d192ad915ec07
|
||||
dependencies:
|
||||
- name: espressif/wiznet_common
|
||||
registry_url: https://components.espressif.com
|
||||
require: private
|
||||
version: ^1.1.0
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=6.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 2.0.1
|
||||
espressif/wiznet_common:
|
||||
component_hash: c32814cb872f0dca09f0c97ab231f9264a65bacc13fab6baf65f71959269a73d
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.3'
|
||||
source:
|
||||
registry_url: https://components.espressif.com
|
||||
type: service
|
||||
version: 1.1.0
|
||||
idf:
|
||||
source:
|
||||
type: idf
|
||||
version: 6.1.0
|
||||
direct_dependencies:
|
||||
- espressif/w5500
|
||||
manifest_hash: 37980d62fb7704a9cc50f484fba5b42932d4093c4bd83087951ae502d64f97c2
|
||||
target: esp32s3
|
||||
version: 3.0.0
|
||||
@@ -0,0 +1,5 @@
|
||||
idf_component_register(SRCS "bridge_main.c" "bridge_eth.c"
|
||||
INCLUDE_DIRS "."
|
||||
REQUIRES esp_wifi esp_netif esp_event nvs_flash esp_driver_usb_serial_jtag esp_timer esp_hw_support
|
||||
esp_driver_i2s esp_driver_gpio esp_driver_spi esp_eth lwip
|
||||
w5500)
|
||||
@@ -0,0 +1,85 @@
|
||||
menu "Bridge Ethernet (W5500)"
|
||||
|
||||
config BRIDGE_ETH
|
||||
bool "W5500 Ethernet with a UDP bridge"
|
||||
default y if IDF_TARGET_ESP32S3
|
||||
default n
|
||||
help
|
||||
The board has a W5500 on SPI, wired to the network. It takes UDP datagrams on BRIDGE_ETH_PORT
|
||||
(dst_mac + packet, sent on ESP-NOW) and sends frames it hears back to the last sender (src_mac + packet).
|
||||
Its address is logged over the USB link once DHCP answers.
|
||||
|
||||
config BRIDGE_ETH_PORT
|
||||
int "UDP port"
|
||||
depends on BRIDGE_ETH
|
||||
default 8766
|
||||
|
||||
config BRIDGE_ETH_AUDIO_PORT
|
||||
int "UDP control port (microphone audio and HELLO)"
|
||||
depends on BRIDGE_ETH
|
||||
default 8767
|
||||
|
||||
config BRIDGE_ETH_SPI_CLOCK_MHZ
|
||||
int "SPI clock (MHz)"
|
||||
depends on BRIDGE_ETH
|
||||
range 5 80
|
||||
default 20
|
||||
|
||||
config BRIDGE_ETH_SCLK_GPIO
|
||||
int "SCLK GPIO (XIAO D8)"
|
||||
depends on BRIDGE_ETH
|
||||
default 7
|
||||
|
||||
config BRIDGE_ETH_MISO_GPIO
|
||||
int "MISO GPIO (XIAO D9)"
|
||||
depends on BRIDGE_ETH
|
||||
default 8
|
||||
|
||||
config BRIDGE_ETH_MOSI_GPIO
|
||||
int "MOSI GPIO (XIAO D10)"
|
||||
depends on BRIDGE_ETH
|
||||
default 9
|
||||
|
||||
config BRIDGE_ETH_CS_GPIO
|
||||
int "CS GPIO (XIAO D1)"
|
||||
depends on BRIDGE_ETH
|
||||
default 2
|
||||
|
||||
config BRIDGE_ETH_INT_GPIO
|
||||
int "INT GPIO (XIAO D2), -1 to poll"
|
||||
depends on BRIDGE_ETH
|
||||
default 3
|
||||
|
||||
config BRIDGE_ETH_RST_GPIO
|
||||
int "RST GPIO (XIAO D0), -1 if not wired"
|
||||
depends on BRIDGE_ETH
|
||||
default 1
|
||||
|
||||
endmenu
|
||||
|
||||
menu "Bridge microphone (I2S)"
|
||||
|
||||
config BRIDGE_MIC
|
||||
bool "Stream an I2S microphone to the host"
|
||||
default y if IDF_TARGET_ESP32S3
|
||||
default n
|
||||
help
|
||||
The host (led-controller on the desktop) can record from an I2S mic on this board
|
||||
(INMP441 wiring, as on the controller's XIAO ESP32-S3). Off on boards without one.
|
||||
|
||||
config BRIDGE_MIC_SCK_GPIO
|
||||
int "I2S SCK (bit clock) GPIO"
|
||||
depends on BRIDGE_MIC
|
||||
default 5
|
||||
|
||||
config BRIDGE_MIC_WS_GPIO
|
||||
int "I2S WS (word select) GPIO"
|
||||
depends on BRIDGE_MIC
|
||||
default 6
|
||||
|
||||
config BRIDGE_MIC_SD_GPIO
|
||||
int "I2S SD (data) GPIO"
|
||||
depends on BRIDGE_MIC
|
||||
default 43
|
||||
|
||||
endmenu
|
||||
@@ -0,0 +1,123 @@
|
||||
#include "bridge_eth.h"
|
||||
|
||||
#include <string.h>
|
||||
#include "driver/gpio.h"
|
||||
#include "driver/spi_master.h"
|
||||
#include "esp_eth.h"
|
||||
#include "esp_eth_mac_w5500.h"
|
||||
#include "esp_eth_phy_w5500.h"
|
||||
#include "esp_event.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_mac.h"
|
||||
#include "esp_netif.h"
|
||||
#include "sdkconfig.h"
|
||||
|
||||
static const char *TAG = "bridge_eth";
|
||||
static volatile bool s_up;
|
||||
static uint8_t s_ip[4];
|
||||
|
||||
static void on_got_ip(void *arg, esp_event_base_t base, int32_t id, void *data)
|
||||
{
|
||||
(void)arg;
|
||||
(void)base;
|
||||
(void)id;
|
||||
const ip_event_got_ip_t *e = data;
|
||||
s_up = true;
|
||||
esp_ip4_addr_t a = e->ip_info.ip;
|
||||
s_ip[0] = esp_ip4_addr1_16(&a);
|
||||
s_ip[1] = esp_ip4_addr2_16(&a);
|
||||
s_ip[2] = esp_ip4_addr3_16(&a);
|
||||
s_ip[3] = esp_ip4_addr4_16(&a);
|
||||
ESP_LOGI(TAG, "Ethernet address " IPSTR "/" IPSTR, IP2STR(&e->ip_info.ip), IP2STR(&e->ip_info.netmask));
|
||||
}
|
||||
|
||||
static void on_eth_down(void *arg, esp_event_base_t base, int32_t id, void *data)
|
||||
{
|
||||
(void)arg;
|
||||
(void)base;
|
||||
(void)id;
|
||||
(void)data;
|
||||
s_up = false;
|
||||
}
|
||||
|
||||
bool bridge_eth_up(void)
|
||||
{
|
||||
return s_up;
|
||||
}
|
||||
|
||||
bool bridge_eth_ip(uint8_t ip[4])
|
||||
{
|
||||
if (!s_up) {
|
||||
return false;
|
||||
}
|
||||
memcpy(ip, s_ip, 4);
|
||||
return true;
|
||||
}
|
||||
|
||||
esp_err_t bridge_eth_start(void)
|
||||
{
|
||||
// The INT pin uses a GPIO interrupt; the service may already be installed by another driver
|
||||
esp_err_t isr = gpio_install_isr_service(0);
|
||||
if (isr != ESP_OK && isr != ESP_ERR_INVALID_STATE) {
|
||||
ESP_LOGE(TAG, "GPIO ISR service: %s", esp_err_to_name(isr));
|
||||
return ESP_FAIL;
|
||||
}
|
||||
const spi_host_device_t host = SPI2_HOST;
|
||||
spi_bus_config_t buscfg = {
|
||||
.miso_io_num = CONFIG_BRIDGE_ETH_MISO_GPIO,
|
||||
.mosi_io_num = CONFIG_BRIDGE_ETH_MOSI_GPIO,
|
||||
.sclk_io_num = CONFIG_BRIDGE_ETH_SCLK_GPIO,
|
||||
.quadwp_io_num = -1,
|
||||
.quadhd_io_num = -1,
|
||||
};
|
||||
if (spi_bus_initialize(host, &buscfg, SPI_DMA_CH_AUTO) != ESP_OK) {
|
||||
ESP_LOGE(TAG, "SPI bus init failed");
|
||||
return ESP_FAIL;
|
||||
}
|
||||
spi_device_interface_config_t devcfg = {
|
||||
.mode = 0,
|
||||
.clock_speed_hz = CONFIG_BRIDGE_ETH_SPI_CLOCK_MHZ * 1000 * 1000,
|
||||
.queue_size = 20,
|
||||
.spics_io_num = CONFIG_BRIDGE_ETH_CS_GPIO,
|
||||
};
|
||||
eth_w5500_config_t wcfg = ETH_W5500_DEFAULT_CONFIG(host, &devcfg);
|
||||
wcfg.base.int_gpio_num = CONFIG_BRIDGE_ETH_INT_GPIO;
|
||||
if (CONFIG_BRIDGE_ETH_INT_GPIO < 0) {
|
||||
wcfg.base.poll_period_ms = 10; // no INT wire: poll the chip
|
||||
}
|
||||
eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG();
|
||||
eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG();
|
||||
phy_config.reset_gpio_num = CONFIG_BRIDGE_ETH_RST_GPIO;
|
||||
esp_eth_mac_t *mac = esp_eth_mac_new_w5500(&wcfg, &mac_config);
|
||||
esp_eth_phy_t *phy = esp_eth_phy_new_w5500(&phy_config);
|
||||
if (!mac || !phy) {
|
||||
ESP_LOGE(TAG, "W5500 MAC/PHY create failed");
|
||||
return ESP_FAIL;
|
||||
}
|
||||
esp_eth_config_t ecfg = ETH_DEFAULT_CONFIG(mac, phy);
|
||||
esp_eth_handle_t eth = NULL;
|
||||
if (esp_eth_driver_install(&ecfg, ð) != ESP_OK) {
|
||||
ESP_LOGE(TAG, "W5500 not found: check its wiring and power");
|
||||
spi_bus_free(host);
|
||||
return ESP_FAIL;
|
||||
}
|
||||
// The W5500 has no factory address: use the board's Ethernet MAC from eFuse
|
||||
uint8_t eth_mac[6];
|
||||
esp_read_mac(eth_mac, ESP_MAC_ETH);
|
||||
esp_eth_ioctl(eth, ETH_CMD_S_MAC_ADDR, eth_mac);
|
||||
|
||||
esp_netif_config_t ncfg = ESP_NETIF_DEFAULT_ETH();
|
||||
esp_netif_t *netif = esp_netif_new(&ncfg);
|
||||
if (!netif || esp_netif_attach(netif, esp_eth_new_netif_glue(eth)) != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Ethernet netif failed");
|
||||
return ESP_FAIL;
|
||||
}
|
||||
esp_event_handler_register(IP_EVENT, IP_EVENT_ETH_GOT_IP, on_got_ip, NULL);
|
||||
esp_event_handler_register(ETH_EVENT, ETHERNET_EVENT_DISCONNECTED, on_eth_down, NULL);
|
||||
if (esp_eth_start(eth) != ESP_OK) {
|
||||
ESP_LOGE(TAG, "Ethernet start failed");
|
||||
return ESP_FAIL;
|
||||
}
|
||||
ESP_LOGI(TAG, "W5500 on SPI, waiting for DHCP");
|
||||
return ESP_OK;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
#pragma once
|
||||
|
||||
// Ethernet for the bridge board: a W5500 on SPI, DHCP. The UDP bridge (bridge_main.c) runs on it: a host sends
|
||||
// dst_mac + packet to this board's port and gets src_mac + packet back, the same as the simulator's bridge.
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#include "esp_err.h"
|
||||
|
||||
// Start the W5500 (pins from Kconfig). Its address is logged once DHCP answers. ESP_FAIL when there is no
|
||||
// W5500 on the board; the USB bridge works without it.
|
||||
esp_err_t bridge_eth_start(void);
|
||||
bool bridge_eth_up(void);
|
||||
// The board's IPv4 address (a.b.c.d) once DHCP has answered; false until then
|
||||
bool bridge_eth_ip(uint8_t ip[4]);
|
||||
@@ -0,0 +1,597 @@
|
||||
/*
|
||||
* USB ESP-NOW bridge. A computer talks to it over the USB-Serial-JTAG port in SLIP frames
|
||||
* (bridge_proto.h); it sends each PKT's dst_mac + packet on ESP-NOW and returns every received frame
|
||||
* as src_mac + packet. ESP-NOW runs on one Wi-Fi channel with no access point and no router.
|
||||
*/
|
||||
#include <stdarg.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "driver/gpio.h"
|
||||
#include "driver/i2s_std.h"
|
||||
#include "driver/usb_serial_jtag.h"
|
||||
#include "esp_log.h"
|
||||
#include "esp_mac.h"
|
||||
#include "esp_netif.h"
|
||||
#include "esp_event.h"
|
||||
#include "esp_now.h"
|
||||
#include "esp_wifi.h"
|
||||
#include "freertos/FreeRTOS.h"
|
||||
#include "freertos/queue.h"
|
||||
#include "freertos/semphr.h"
|
||||
#include "esp_timer.h"
|
||||
#include "freertos/task.h"
|
||||
#include "lwip/sockets.h"
|
||||
#include "nvs.h"
|
||||
#include "nvs_flash.h"
|
||||
#include "bridge_proto.h"
|
||||
#include "sdkconfig.h"
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
#include "bridge_eth.h"
|
||||
#endif
|
||||
|
||||
static const char *TAG = "bridge";
|
||||
|
||||
#define DEFAULT_CHANNEL 6 // the controller's CONFIG_ESPNOW_CHANNEL
|
||||
#define PEERS_MAX 19 // ESP-NOW allows 20 peers: 19 unicast and the broadcast one
|
||||
#define TX_QUEUE_LEN 32
|
||||
#define RX_QUEUE_LEN 32
|
||||
#define SEND_WAIT_MS 100
|
||||
|
||||
typedef struct {
|
||||
uint8_t mac[6];
|
||||
uint8_t len;
|
||||
uint8_t data[BRIDGE_PKT_MAX];
|
||||
} pkt_t;
|
||||
|
||||
static const uint8_t BROADCAST[6] = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF };
|
||||
|
||||
static QueueHandle_t s_txq;
|
||||
static QueueHandle_t s_rxq;
|
||||
static SemaphoreHandle_t s_sent; // given by the ESP-NOW send callback
|
||||
static SemaphoreHandle_t s_usb_lock;
|
||||
static volatile esp_now_send_status_t s_last_status;
|
||||
static volatile bool s_usb_ready;
|
||||
|
||||
static uint8_t s_mac[6];
|
||||
static uint8_t s_channel = DEFAULT_CHANNEL;
|
||||
static uint8_t s_peers[PEERS_MAX][6]; // most recently used first
|
||||
static int s_npeers;
|
||||
|
||||
static uint32_t s_tx, s_tx_fail, s_rx, s_drop;
|
||||
|
||||
static volatile bool s_mic_on; // the host asked for microphone audio (SET_MIC)
|
||||
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
// The UDP bridge: the host that sent the last datagram gets every frame the radio hears
|
||||
static int s_udp = -1;
|
||||
static SemaphoreHandle_t s_udp_lock;
|
||||
static struct sockaddr_in s_udp_peer;
|
||||
static bool s_udp_have_peer;
|
||||
|
||||
// The control port (BRIDGE_ETH_AUDIO_PORT): HELLO gets INFO back; SET_MIC 1 from a host makes this board send it
|
||||
// microphone audio (AUDIO datagrams) until SET_MIC 1 stops arriving for MIC_UDP_TIMEOUT_MS
|
||||
#define MIC_UDP_TIMEOUT_MS 2000
|
||||
static int s_ctl = -1;
|
||||
static struct sockaddr_in s_ctl_peer;
|
||||
static bool s_ctl_mic;
|
||||
static int64_t s_ctl_mic_ms;
|
||||
#endif
|
||||
|
||||
/* ---- USB link ---------------------------------------------------------------------------------- */
|
||||
|
||||
// One whole frame per call, under the lock so frames from different tasks never interleave
|
||||
#define USB_WRITE_PIECE 64
|
||||
static void usb_send_frame(uint8_t type, const uint8_t *body, size_t n)
|
||||
{
|
||||
static uint8_t out[BRIDGE_ENC_MAX];
|
||||
if (!s_usb_ready) {
|
||||
return;
|
||||
}
|
||||
size_t len = bridge_encode(out, sizeof(out), type, body, n);
|
||||
if (!len) {
|
||||
return;
|
||||
}
|
||||
xSemaphoreTake(s_usb_lock, portMAX_DELAY);
|
||||
size_t off = 0;
|
||||
while (off < len) {
|
||||
// The driver's TX ring takes a write only as a whole, so a frame longer than its ring is written in pieces
|
||||
size_t piece = len - off < USB_WRITE_PIECE ? len - off : USB_WRITE_PIECE;
|
||||
int w = usb_serial_jtag_write_bytes(out + off, piece, pdMS_TO_TICKS(200));
|
||||
if (w <= 0) {
|
||||
break; // host not reading: drop the rest rather than stall the radio
|
||||
}
|
||||
off += (size_t)w;
|
||||
}
|
||||
xSemaphoreGive(s_usb_lock);
|
||||
}
|
||||
|
||||
// Log lines travel as LOG frames, so nothing but frames ever reaches the USB port
|
||||
static int log_hook(const char *fmt, va_list args)
|
||||
{
|
||||
char line[200];
|
||||
int n = vsnprintf(line, sizeof(line), fmt, args);
|
||||
if (n > 0) {
|
||||
size_t len = (size_t)n < sizeof(line) ? (size_t)n : sizeof(line) - 1;
|
||||
usb_send_frame(BRIDGE_MSG_LOG, (const uint8_t *)line, len);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
#if CONFIG_BRIDGE_MIC
|
||||
#define BRIDGE_CAP_MIC_BOARD BRIDGE_CAP_MIC
|
||||
#else
|
||||
#define BRIDGE_CAP_MIC_BOARD 0
|
||||
#endif
|
||||
|
||||
static void put_u32(uint8_t *p, uint32_t v)
|
||||
{
|
||||
p[0] = v & 0xFF;
|
||||
p[1] = (v >> 8) & 0xFF;
|
||||
p[2] = (v >> 16) & 0xFF;
|
||||
p[3] = (v >> 24) & 0xFF;
|
||||
}
|
||||
|
||||
// The INFO body (see bridge_proto.h), the same over USB and over the UDP control port
|
||||
static void build_info(uint8_t *b)
|
||||
{
|
||||
b[0] = BRIDGE_PROTO_VERSION;
|
||||
memcpy(b + 1, s_mac, 6);
|
||||
b[7] = s_channel;
|
||||
put_u32(b + 8, s_tx);
|
||||
put_u32(b + 12, s_tx_fail);
|
||||
put_u32(b + 16, s_rx);
|
||||
put_u32(b + 20, s_drop);
|
||||
uint8_t ip[4] = { 0, 0, 0, 0 };
|
||||
uint8_t eth_cap = 0;
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
if (bridge_eth_ip(ip)) {
|
||||
eth_cap = BRIDGE_CAP_ETH;
|
||||
}
|
||||
#endif
|
||||
b[24] = BRIDGE_CAP_MIC_BOARD | eth_cap;
|
||||
memcpy(b + 25, ip, 4);
|
||||
}
|
||||
|
||||
static void send_info(void)
|
||||
{
|
||||
uint8_t b[BRIDGE_INFO_LEN];
|
||||
build_info(b);
|
||||
usb_send_frame(BRIDGE_MSG_INFO, b, sizeof(b));
|
||||
}
|
||||
|
||||
/* ---- ESP-NOW ----------------------------------------------------------------------------------- */
|
||||
|
||||
static void on_sent(const esp_now_send_info_t *info, esp_now_send_status_t status)
|
||||
{
|
||||
(void)info;
|
||||
s_last_status = status;
|
||||
xSemaphoreGive(s_sent);
|
||||
}
|
||||
|
||||
static void on_recv(const esp_now_recv_info_t *info, const uint8_t *data, int len)
|
||||
{
|
||||
if (len <= 0 || len > BRIDGE_PKT_MAX) {
|
||||
return;
|
||||
}
|
||||
pkt_t p;
|
||||
memcpy(p.mac, info->src_addr, 6);
|
||||
p.len = (uint8_t)len;
|
||||
memcpy(p.data, data, len);
|
||||
if (xQueueSend(s_rxq, &p, 0) == pdTRUE) {
|
||||
s_rx++;
|
||||
} else {
|
||||
s_drop++;
|
||||
}
|
||||
}
|
||||
|
||||
// Keep the peer table within the ESP-NOW limit, least recently used dropped first
|
||||
static void ensure_peer(const uint8_t *mac)
|
||||
{
|
||||
if (memcmp(mac, BROADCAST, 6) == 0) {
|
||||
if (!esp_now_is_peer_exist(BROADCAST)) {
|
||||
esp_now_peer_info_t p = { .channel = s_channel, .ifidx = WIFI_IF_STA, .encrypt = false };
|
||||
memcpy(p.peer_addr, BROADCAST, 6);
|
||||
esp_now_add_peer(&p);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (int i = 0; i < s_npeers; i++) {
|
||||
if (memcmp(s_peers[i], mac, 6) == 0) {
|
||||
uint8_t hit[6];
|
||||
memcpy(hit, mac, 6);
|
||||
memmove(s_peers[1], s_peers[0], i * 6);
|
||||
memcpy(s_peers[0], hit, 6);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (s_npeers == PEERS_MAX) {
|
||||
esp_now_del_peer(s_peers[PEERS_MAX - 1]);
|
||||
s_npeers--;
|
||||
}
|
||||
esp_now_peer_info_t p = { .channel = s_channel, .ifidx = WIFI_IF_STA, .encrypt = false };
|
||||
memcpy(p.peer_addr, mac, 6);
|
||||
if (esp_now_add_peer(&p) != ESP_OK) {
|
||||
return;
|
||||
}
|
||||
memmove(s_peers[1], s_peers[0], s_npeers * 6);
|
||||
memcpy(s_peers[0], mac, 6);
|
||||
s_npeers++;
|
||||
}
|
||||
|
||||
static void drop_unicast_peers(void)
|
||||
{
|
||||
for (int i = 0; i < s_npeers; i++) {
|
||||
esp_now_del_peer(s_peers[i]);
|
||||
}
|
||||
s_npeers = 0;
|
||||
}
|
||||
|
||||
static void set_channel(uint8_t ch)
|
||||
{
|
||||
drop_unicast_peers(); // peers were added on the old channel
|
||||
s_channel = ch;
|
||||
esp_wifi_set_channel(ch, WIFI_SECOND_CHAN_NONE);
|
||||
nvs_handle_t h;
|
||||
if (nvs_open("bridge", NVS_READWRITE, &h) == ESP_OK) {
|
||||
nvs_set_u8(h, "channel", ch);
|
||||
nvs_commit(h);
|
||||
nvs_close(h);
|
||||
}
|
||||
ESP_LOGI(TAG, "ESP-NOW channel %u", ch);
|
||||
}
|
||||
|
||||
// One frame at a time, waiting for each send callback, so the radio is never overrun
|
||||
static void tx_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
pkt_t p;
|
||||
for (;;) {
|
||||
if (xQueueReceive(s_txq, &p, portMAX_DELAY) != pdTRUE) {
|
||||
continue;
|
||||
}
|
||||
ensure_peer(p.mac);
|
||||
if (esp_now_send(p.mac, p.data, p.len) != ESP_OK) {
|
||||
s_tx_fail++;
|
||||
continue;
|
||||
}
|
||||
if (xSemaphoreTake(s_sent, pdMS_TO_TICKS(SEND_WAIT_MS)) == pdTRUE && s_last_status == ESP_NOW_SEND_SUCCESS) {
|
||||
s_tx++;
|
||||
} else {
|
||||
s_tx_fail++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
static void udp_send_up(const uint8_t *body, size_t n);
|
||||
#endif
|
||||
|
||||
// Received frames go to the host as src_mac + packet
|
||||
static void rx_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
pkt_t p;
|
||||
uint8_t body[6 + BRIDGE_PKT_MAX];
|
||||
for (;;) {
|
||||
if (xQueueReceive(s_rxq, &p, portMAX_DELAY) != pdTRUE) {
|
||||
continue;
|
||||
}
|
||||
memcpy(body, p.mac, 6);
|
||||
memcpy(body + 6, p.data, p.len);
|
||||
usb_send_frame(BRIDGE_MSG_PKT, body, 6 + p.len);
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
udp_send_up(body, 6 + p.len);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
// Frames heard on ESP-NOW go to the last UDP sender as src_mac + packet
|
||||
static void udp_send_up(const uint8_t *body, size_t n)
|
||||
{
|
||||
xSemaphoreTake(s_udp_lock, portMAX_DELAY);
|
||||
if (s_udp_have_peer && s_udp >= 0) {
|
||||
sendto(s_udp, body, n, 0, (const struct sockaddr *)&s_udp_peer, sizeof(s_udp_peer));
|
||||
}
|
||||
xSemaphoreGive(s_udp_lock);
|
||||
}
|
||||
|
||||
// Datagrams from a host are dst_mac + packet, queued like a USB PKT frame. The sender becomes the peer.
|
||||
static void udp_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
uint8_t buf[6 + BRIDGE_PKT_MAX];
|
||||
for (;;) {
|
||||
struct sockaddr_in from;
|
||||
socklen_t fl = sizeof(from);
|
||||
int n = recvfrom(s_udp, buf, sizeof(buf), 0, (struct sockaddr *)&from, &fl);
|
||||
if (n <= 6) {
|
||||
continue;
|
||||
}
|
||||
xSemaphoreTake(s_udp_lock, portMAX_DELAY);
|
||||
s_udp_peer = from;
|
||||
s_udp_have_peer = true;
|
||||
xSemaphoreGive(s_udp_lock);
|
||||
pkt_t p;
|
||||
memcpy(p.mac, buf, 6);
|
||||
p.len = (uint8_t)(n - 6);
|
||||
memcpy(p.data, buf + 6, p.len);
|
||||
if (xQueueSend(s_txq, &p, 0) != pdTRUE) {
|
||||
s_drop++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
static void ctl_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
uint8_t buf[32];
|
||||
for (;;) {
|
||||
struct sockaddr_in from;
|
||||
socklen_t fl = sizeof(from);
|
||||
int n = recvfrom(s_ctl, buf, sizeof(buf), 0, (struct sockaddr *)&from, &fl);
|
||||
if (n < 1) {
|
||||
vTaskDelay(pdMS_TO_TICKS(10));
|
||||
continue;
|
||||
}
|
||||
if (buf[0] == BRIDGE_MSG_HELLO) {
|
||||
uint8_t out[1 + BRIDGE_INFO_LEN];
|
||||
out[0] = BRIDGE_MSG_INFO;
|
||||
build_info(out + 1);
|
||||
sendto(s_ctl, out, sizeof(out), 0, (struct sockaddr *)&from, fl);
|
||||
} else if (buf[0] == BRIDGE_MSG_SET_MIC && n >= 2) {
|
||||
xSemaphoreTake(s_udp_lock, portMAX_DELAY);
|
||||
s_ctl_peer = from;
|
||||
s_ctl_mic = buf[1] != 0;
|
||||
s_ctl_mic_ms = esp_timer_get_time() / 1000;
|
||||
xSemaphoreGive(s_udp_lock);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void ctl_open(void)
|
||||
{
|
||||
s_ctl = socket(AF_INET, SOCK_DGRAM, 0);
|
||||
struct sockaddr_in a = { .sin_family = AF_INET, .sin_port = htons(CONFIG_BRIDGE_ETH_AUDIO_PORT), .sin_addr.s_addr = htonl(INADDR_ANY) };
|
||||
if (s_ctl < 0 || bind(s_ctl, (struct sockaddr *)&a, sizeof(a)) != 0) {
|
||||
ESP_LOGE(TAG, "UDP control port %d not open", CONFIG_BRIDGE_ETH_AUDIO_PORT);
|
||||
return;
|
||||
}
|
||||
xTaskCreate(ctl_task, "udp_ctl", 4096, NULL, 5, NULL);
|
||||
ESP_LOGI(TAG, "UDP control on port %d", CONFIG_BRIDGE_ETH_AUDIO_PORT);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void udp_open(void)
|
||||
{
|
||||
ESP_LOGI(TAG, "udp: opening");
|
||||
s_udp = socket(AF_INET, SOCK_DGRAM, 0);
|
||||
ESP_LOGI(TAG, "udp: socket %d", s_udp);
|
||||
struct sockaddr_in a = { .sin_family = AF_INET, .sin_port = htons(CONFIG_BRIDGE_ETH_PORT), .sin_addr.s_addr = htonl(INADDR_ANY) };
|
||||
if (s_udp < 0 || bind(s_udp, (struct sockaddr *)&a, sizeof(a)) != 0) {
|
||||
ESP_LOGE(TAG, "UDP port %d not open", CONFIG_BRIDGE_ETH_PORT);
|
||||
return;
|
||||
}
|
||||
xTaskCreate(udp_task, "udp_rx", 4096, NULL, 5, NULL);
|
||||
ESP_LOGI(TAG, "UDP bridge on port %d", CONFIG_BRIDGE_ETH_PORT);
|
||||
ctl_open();
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ---- host commands ----------------------------------------------------------------------------- */
|
||||
|
||||
static void handle_frame(const uint8_t *msg, size_t len)
|
||||
{
|
||||
if (len < 1) {
|
||||
return;
|
||||
}
|
||||
const uint8_t *body = msg + 1;
|
||||
size_t n = len - 1;
|
||||
switch (msg[0]) {
|
||||
case BRIDGE_MSG_PKT: {
|
||||
if (n < 7 || n > 6 + BRIDGE_PKT_MAX) {
|
||||
return;
|
||||
}
|
||||
pkt_t p;
|
||||
memcpy(p.mac, body, 6);
|
||||
p.len = (uint8_t)(n - 6);
|
||||
memcpy(p.data, body + 6, p.len);
|
||||
if (xQueueSend(s_txq, &p, 0) != pdTRUE) {
|
||||
s_drop++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case BRIDGE_MSG_HELLO:
|
||||
send_info();
|
||||
break;
|
||||
case BRIDGE_MSG_SET_CHANNEL:
|
||||
if (n >= 1 && body[0] >= 1 && body[0] <= 13) {
|
||||
set_channel(body[0]);
|
||||
}
|
||||
send_info();
|
||||
break;
|
||||
case BRIDGE_MSG_SET_MIC:
|
||||
if (n >= 1) {
|
||||
s_mic_on = body[0] != 0;
|
||||
ESP_LOGI(TAG, "microphone %s", s_mic_on ? "on" : "off");
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static void host_rx_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
uint8_t buf[64];
|
||||
bridge_decoder_t dec = { 0 };
|
||||
for (;;) {
|
||||
int n = usb_serial_jtag_read_bytes(buf, sizeof(buf), pdMS_TO_TICKS(50));
|
||||
for (int i = 0; i < n; i++) {
|
||||
if (bridge_decode_feed(&dec, buf[i])) {
|
||||
handle_frame(dec.buf, dec.frame_len);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ---- microphone (boards with BRIDGE_MIC) ------------------------------------------------------- */
|
||||
|
||||
#if CONFIG_BRIDGE_MIC
|
||||
#define MIC_RATE 16000
|
||||
static i2s_chan_handle_t s_mic_chan;
|
||||
|
||||
// Reads the I2S mic all the time (so the bus never stalls) and sends 10 ms of 16-bit mono while the host wants it.
|
||||
// The mic drives one of the two slots and the other reads 0: the busier slot is the one kept.
|
||||
static void mic_task(void *arg)
|
||||
{
|
||||
(void)arg;
|
||||
static int32_t raw[BRIDGE_AUDIO_SAMPLES * 2];
|
||||
uint8_t out[BRIDGE_AUDIO_BYTES];
|
||||
uint8_t dg[1 + BRIDGE_AUDIO_BYTES]; // a UDP audio datagram: AUDIO, then the samples
|
||||
for (;;) {
|
||||
size_t got = 0;
|
||||
esp_err_t err = i2s_channel_read(s_mic_chan, raw, sizeof(raw), &got, portMAX_DELAY);
|
||||
if (err != ESP_OK || got == 0) {
|
||||
continue;
|
||||
}
|
||||
size_t frames = got / (2 * sizeof(int32_t));
|
||||
size_t nonzero[2] = { 0, 0 };
|
||||
for (size_t i = 0; i < frames; i++) {
|
||||
nonzero[0] += raw[2 * i] != 0;
|
||||
nonzero[1] += raw[2 * i + 1] != 0;
|
||||
}
|
||||
int ch = nonzero[1] > nonzero[0] ? 1 : 0;
|
||||
bool want_usb = s_mic_on;
|
||||
bool want_udp = false;
|
||||
struct sockaddr_in peer;
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
xSemaphoreTake(s_udp_lock, portMAX_DELAY);
|
||||
want_udp = s_ctl_mic && esp_timer_get_time() / 1000 - s_ctl_mic_ms < MIC_UDP_TIMEOUT_MS;
|
||||
peer = s_ctl_peer;
|
||||
xSemaphoreGive(s_udp_lock);
|
||||
#endif
|
||||
if (!want_usb && !want_udp) {
|
||||
continue;
|
||||
}
|
||||
for (size_t i = 0; i < frames; i++) {
|
||||
int16_t s = (int16_t)(raw[2 * i + ch] >> 16); // 24-bit sample in the top of the 32-bit slot
|
||||
out[2 * i] = (uint8_t)(s & 0xFF);
|
||||
out[2 * i + 1] = (uint8_t)((uint16_t)s >> 8);
|
||||
}
|
||||
if (want_usb) {
|
||||
usb_send_frame(BRIDGE_MSG_AUDIO, out, 2 * frames);
|
||||
}
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
if (want_udp) {
|
||||
dg[0] = BRIDGE_MSG_AUDIO;
|
||||
memcpy(dg + 1, out, 2 * frames);
|
||||
sendto(s_ctl, dg, 1 + 2 * frames, 0, (struct sockaddr *)&peer, sizeof(peer));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
static void mic_start(void)
|
||||
{
|
||||
i2s_chan_config_t chan = I2S_CHANNEL_DEFAULT_CONFIG(I2S_NUM_0, I2S_ROLE_MASTER);
|
||||
if (i2s_new_channel(&chan, NULL, &s_mic_chan) != ESP_OK) {
|
||||
ESP_LOGE(TAG, "I2S channel create failed: no microphone");
|
||||
return;
|
||||
}
|
||||
i2s_std_config_t std = {
|
||||
.clk_cfg = I2S_STD_CLK_DEFAULT_CONFIG(MIC_RATE),
|
||||
.slot_cfg = I2S_STD_PHILIPS_SLOT_DEFAULT_CONFIG(I2S_DATA_BIT_WIDTH_32BIT, I2S_SLOT_MODE_STEREO),
|
||||
.gpio_cfg = {
|
||||
.mclk = I2S_GPIO_UNUSED,
|
||||
.bclk = CONFIG_BRIDGE_MIC_SCK_GPIO,
|
||||
.ws = CONFIG_BRIDGE_MIC_WS_GPIO,
|
||||
.dout = I2S_GPIO_UNUSED,
|
||||
.din = CONFIG_BRIDGE_MIC_SD_GPIO,
|
||||
},
|
||||
};
|
||||
if (i2s_channel_init_std_mode(s_mic_chan, &std) != ESP_OK || i2s_channel_enable(s_mic_chan) != ESP_OK) {
|
||||
ESP_LOGE(TAG, "I2S mic init failed: no microphone");
|
||||
return;
|
||||
}
|
||||
gpio_pulldown_en(CONFIG_BRIDGE_MIC_SD_GPIO); // SD tri-states outside the mic's slot
|
||||
xTaskCreate(mic_task, "mic", 4096, NULL, 5, NULL);
|
||||
ESP_LOGI(TAG, "I2S mic on SCK %d WS %d SD %d at %d Hz",
|
||||
CONFIG_BRIDGE_MIC_SCK_GPIO, CONFIG_BRIDGE_MIC_WS_GPIO, CONFIG_BRIDGE_MIC_SD_GPIO, MIC_RATE);
|
||||
}
|
||||
#else
|
||||
static void mic_start(void)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ---- start-up ---------------------------------------------------------------------------------- */
|
||||
|
||||
static void load_channel(void)
|
||||
{
|
||||
nvs_handle_t h;
|
||||
if (nvs_open("bridge", NVS_READONLY, &h) == ESP_OK) {
|
||||
uint8_t ch = 0;
|
||||
if (nvs_get_u8(h, "channel", &ch) == ESP_OK && ch >= 1 && ch <= 13) {
|
||||
s_channel = ch;
|
||||
}
|
||||
nvs_close(h);
|
||||
}
|
||||
}
|
||||
|
||||
void app_main(void)
|
||||
{
|
||||
esp_err_t err = nvs_flash_init();
|
||||
if (err == ESP_ERR_NVS_NO_FREE_PAGES || err == ESP_ERR_NVS_NEW_VERSION_FOUND) {
|
||||
nvs_flash_erase();
|
||||
err = nvs_flash_init();
|
||||
}
|
||||
ESP_ERROR_CHECK(err);
|
||||
load_channel();
|
||||
|
||||
s_txq = xQueueCreate(TX_QUEUE_LEN, sizeof(pkt_t));
|
||||
s_rxq = xQueueCreate(RX_QUEUE_LEN, sizeof(pkt_t));
|
||||
s_sent = xSemaphoreCreateBinary();
|
||||
s_usb_lock = xSemaphoreCreateMutex();
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
s_udp_lock = xSemaphoreCreateMutex(); // the microphone task uses it from the start
|
||||
#endif
|
||||
|
||||
ESP_ERROR_CHECK(esp_netif_init());
|
||||
ESP_ERROR_CHECK(esp_event_loop_create_default());
|
||||
wifi_init_config_t wcfg = WIFI_INIT_CONFIG_DEFAULT();
|
||||
ESP_ERROR_CHECK(esp_wifi_init(&wcfg));
|
||||
ESP_ERROR_CHECK(esp_wifi_set_storage(WIFI_STORAGE_RAM));
|
||||
ESP_ERROR_CHECK(esp_wifi_set_mode(WIFI_MODE_STA));
|
||||
ESP_ERROR_CHECK(esp_wifi_start());
|
||||
ESP_ERROR_CHECK(esp_wifi_set_ps(WIFI_PS_NONE));
|
||||
ESP_ERROR_CHECK(esp_wifi_set_channel(s_channel, WIFI_SECOND_CHAN_NONE));
|
||||
ESP_ERROR_CHECK(esp_wifi_get_mac(WIFI_IF_STA, s_mac));
|
||||
|
||||
ESP_ERROR_CHECK(esp_now_init());
|
||||
ESP_ERROR_CHECK(esp_now_register_send_cb(on_sent));
|
||||
ESP_ERROR_CHECK(esp_now_register_recv_cb(on_recv));
|
||||
ensure_peer(BROADCAST);
|
||||
|
||||
usb_serial_jtag_driver_config_t ucfg = USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT();
|
||||
ESP_ERROR_CHECK(usb_serial_jtag_driver_install(&ucfg));
|
||||
s_usb_ready = true;
|
||||
esp_log_set_vprintf(log_hook);
|
||||
|
||||
xTaskCreate(tx_task, "espnow_tx", 4096, NULL, 6, NULL);
|
||||
xTaskCreate(rx_task, "espnow_rx", 4096, NULL, 6, NULL);
|
||||
xTaskCreate(host_rx_task, "usb_rx", 4096, NULL, 5, NULL);
|
||||
mic_start();
|
||||
#if CONFIG_BRIDGE_ETH
|
||||
ESP_LOGI(TAG, "eth: starting");
|
||||
esp_err_t eth_err = bridge_eth_start();
|
||||
ESP_LOGI(TAG, "eth: %s", esp_err_to_name(eth_err));
|
||||
if (eth_err == ESP_OK) {
|
||||
udp_open();
|
||||
}
|
||||
#endif
|
||||
ESP_LOGI(TAG, "bridge %02x:%02x:%02x:%02x:%02x:%02x on ESP-NOW channel %u",
|
||||
s_mac[0], s_mac[1], s_mac[2], s_mac[3], s_mac[4], s_mac[5], s_channel);
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* USB ESP-NOW bridge framing, shared by bridge/ (ESP32-C3 firmware) and main/bridge_serial.c (host).
|
||||
*
|
||||
* Each message is one SLIP frame (END-delimited, ESC-escaped) holding [type][body][crc16 LE], where the
|
||||
* CRC-16/CCITT-FALSE covers type and body. Bytes outside frames are ignored by the host.
|
||||
*
|
||||
* 00 PKT host -> bridge dst_mac(6) + packet sent on ESP-NOW to dst_mac
|
||||
* 00 PKT bridge -> host src_mac(6) + packet every frame received on ESP-NOW
|
||||
* 10 HELLO host -> bridge (empty) answered with INFO
|
||||
* 11 INFO bridge -> host see bridge_info below
|
||||
* 12 SET_CHANNEL host -> bridge channel(1) sets the ESP-NOW channel, answered with INFO
|
||||
* 13 SET_MIC host -> bridge on(1) start (1) or stop (0) streaming the microphone
|
||||
* 40 AUDIO bridge -> host BRIDGE_AUDIO_SAMPLES 16-bit LE mono samples at 16 kHz, while the mic is on
|
||||
* 30 LOG bridge -> host text the bridge's log lines
|
||||
*
|
||||
* INFO: byte 24 is capabilities (bit 0 = a microphone, BRIDGE_CAP_MIC; bit 1 = Ethernet up, BRIDGE_CAP_ETH),
|
||||
* bytes 25-28 the board's Ethernet IPv4 address (0.0.0.0 while it has none).
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define BRIDGE_PROTO_VERSION 1
|
||||
#define BRIDGE_PKT_MAX 250 // ESP-NOW payload limit
|
||||
#define BRIDGE_AUDIO_SAMPLES 160 // 10 ms at 16 kHz
|
||||
#define BRIDGE_AUDIO_BYTES (2 * BRIDGE_AUDIO_SAMPLES)
|
||||
#define BRIDGE_PKT_BODY (6 + BRIDGE_PKT_MAX)
|
||||
#define BRIDGE_BODY_MAX (BRIDGE_PKT_BODY > BRIDGE_AUDIO_BYTES ? BRIDGE_PKT_BODY : BRIDGE_AUDIO_BYTES)
|
||||
#define BRIDGE_FRAME_MAX (BRIDGE_BODY_MAX + 3) // type + body + crc
|
||||
#define BRIDGE_ENC_MAX (2 * BRIDGE_FRAME_MAX + 2)
|
||||
#define BRIDGE_INFO_LEN 29
|
||||
#define BRIDGE_CAP_MIC 0x01
|
||||
#define BRIDGE_CAP_ETH 0x02
|
||||
|
||||
enum {
|
||||
BRIDGE_MSG_PKT = 0x00,
|
||||
BRIDGE_MSG_HELLO = 0x10,
|
||||
BRIDGE_MSG_INFO = 0x11,
|
||||
BRIDGE_MSG_SET_CHANNEL = 0x12,
|
||||
BRIDGE_MSG_SET_MIC = 0x13,
|
||||
BRIDGE_MSG_AUDIO = 0x40,
|
||||
BRIDGE_MSG_LOG = 0x30,
|
||||
};
|
||||
|
||||
#define SLIP_END 0xC0
|
||||
#define SLIP_ESC 0xDB
|
||||
#define SLIP_ESC_END 0xDC
|
||||
#define SLIP_ESC_ESC 0xDD
|
||||
|
||||
static inline uint16_t bridge_crc16_update(uint16_t crc, uint8_t b)
|
||||
{
|
||||
crc ^= (uint16_t)b << 8;
|
||||
for (int i = 0; i < 8; i++) {
|
||||
crc = (crc & 0x8000) ? (uint16_t)((crc << 1) ^ 0x1021) : (uint16_t)(crc << 1);
|
||||
}
|
||||
return crc;
|
||||
}
|
||||
|
||||
/* Encode one message into out (capacity cap). Returns the frame length, or 0 if it does not fit. */
|
||||
static inline size_t bridge_encode(uint8_t *out, size_t cap, uint8_t type, const uint8_t *body, size_t n)
|
||||
{
|
||||
size_t off = 0;
|
||||
uint16_t crc = 0xFFFF;
|
||||
if (cap < 2) {
|
||||
return 0;
|
||||
}
|
||||
out[off++] = SLIP_END;
|
||||
/* the type and body bytes, then the CRC, each escaped as they go out */
|
||||
for (size_t i = 0; i < n + 3; i++) {
|
||||
uint8_t b;
|
||||
if (i == 0) {
|
||||
b = type;
|
||||
crc = bridge_crc16_update(crc, b);
|
||||
} else if (i <= n) {
|
||||
b = body[i - 1];
|
||||
crc = bridge_crc16_update(crc, b);
|
||||
} else {
|
||||
b = (i == n + 1) ? (uint8_t)(crc & 0xFF) : (uint8_t)(crc >> 8);
|
||||
}
|
||||
if (b == SLIP_END) {
|
||||
if (off + 2 >= cap) return 0;
|
||||
out[off++] = SLIP_ESC;
|
||||
out[off++] = SLIP_ESC_END;
|
||||
} else if (b == SLIP_ESC) {
|
||||
if (off + 2 >= cap) return 0;
|
||||
out[off++] = SLIP_ESC;
|
||||
out[off++] = SLIP_ESC_ESC;
|
||||
} else {
|
||||
if (off + 1 >= cap) return 0;
|
||||
out[off++] = b;
|
||||
}
|
||||
}
|
||||
if (off + 1 > cap) return 0;
|
||||
out[off++] = SLIP_END;
|
||||
return off;
|
||||
}
|
||||
|
||||
/* Incremental decoder. Feed bytes; when bridge_decode_feed returns true, d->buf holds [type][body] of
|
||||
* length d->frame_len, valid until the next call. */
|
||||
typedef struct {
|
||||
uint8_t buf[BRIDGE_FRAME_MAX];
|
||||
size_t len;
|
||||
size_t frame_len;
|
||||
bool esc;
|
||||
bool overflow;
|
||||
} bridge_decoder_t;
|
||||
|
||||
static inline bool bridge_decode_feed(bridge_decoder_t *d, uint8_t b)
|
||||
{
|
||||
if (b == SLIP_END) {
|
||||
bool ok = false;
|
||||
if (!d->overflow && d->len >= 3) {
|
||||
uint16_t crc = 0xFFFF;
|
||||
for (size_t i = 0; i < d->len - 2; i++) {
|
||||
crc = bridge_crc16_update(crc, d->buf[i]);
|
||||
}
|
||||
uint16_t got = (uint16_t)(d->buf[d->len - 2] | (d->buf[d->len - 1] << 8));
|
||||
ok = crc == got;
|
||||
if (ok) {
|
||||
d->frame_len = d->len - 2;
|
||||
}
|
||||
}
|
||||
d->len = 0;
|
||||
d->esc = false;
|
||||
d->overflow = false;
|
||||
return ok;
|
||||
}
|
||||
if (d->esc) {
|
||||
d->esc = false;
|
||||
if (b == SLIP_ESC_END) {
|
||||
b = SLIP_END;
|
||||
} else if (b == SLIP_ESC_ESC) {
|
||||
b = SLIP_ESC;
|
||||
} else {
|
||||
d->overflow = true; // not valid SLIP: drop the frame
|
||||
}
|
||||
} else if (b == SLIP_ESC) {
|
||||
d->esc = true;
|
||||
return false;
|
||||
}
|
||||
if (d->len < sizeof(d->buf)) {
|
||||
d->buf[d->len++] = b;
|
||||
} else {
|
||||
d->overflow = true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
dependencies:
|
||||
espressif/w5500:
|
||||
version: "^2.0.1"
|
||||
@@ -0,0 +1,7 @@
|
||||
# The USB-Serial-JTAG port carries the SLIP link to the host, so the console must not write to it
|
||||
CONFIG_ESP_CONSOLE_NONE=y
|
||||
CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG=n
|
||||
CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=10
|
||||
CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=32
|
||||
CONFIG_ETH_USE_SPI_ETHERNET=y
|
||||
CONFIG_ETH_SPI_ETHERNET_W5500=y
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"enabled": true,
|
||||
"device": "simulated",
|
||||
"device_override": "",
|
||||
"device_select": "simulated"
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
[
|
||||
{
|
||||
"id": "c9ab898d-740",
|
||||
"label": "bridge1",
|
||||
"transport": "ip",
|
||||
"ip": "192.168.4.1",
|
||||
"ws_port": 80
|
||||
},
|
||||
{
|
||||
"id": "18d85e86-17b",
|
||||
"label": "bridge2",
|
||||
"transport": "serial",
|
||||
"serial_port": "/dev/ttyUSB0",
|
||||
"serial_baudrate": 115200
|
||||
}
|
||||
]
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"188b0e1560a8": {"id": "188b0e1560a8", "name": "led-188b0e1560a8", "type": "led", "transport": "wifi", "address": "10.1.1.192", "default_pattern": null, "zones": []}, "f0f5bdfb9d30": {"id": "f0f5bdfb9d30", "name": "led-f0f5bdfb9d30", "type": "led", "transport": "wifi", "address": "10.1.1.232", "default_pattern": null, "zones": []}}
|
||||
{}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"1": {"name": "Main Group", "devices": ["1", "2", "3"]}, "2": {"name": "Accent Group", "devices": ["4", "5"]}}
|
||||
{"1": {"name": "default", "devices": [], "wifi_driver_display_name": null, "wifi_driver_num_leds": null, "wifi_color_order": null, "wifi_startup_mode": null, "output_brightness": 255, "bridge_id": null, "pattern": "on", "colors": ["000000", "FF0000"], "brightness": 100, "delay": 100, "step_offset": 0, "step_increment": 1, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "id": "1"}}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"1": ["#FF0000", "#00FF00", "#0000FF", "#FFFF00", "#FF00FF", "#00FFFF", "#FFFFFF", "#000000"], "2": [], "3": [], "4": [], "5": [], "6": [], "7": ["#FF0000", "#00FF00", "#0000FF", "#FFFF00", "#FF00FF", "#00FFFF", "#FFFFFF", "#000000"], "8": [], "9": [], "10": [], "11": [], "12": ["#890b0b", "#0b8935"]}
|
||||
{"1": ["#FF0000", "#00FF00", "#0000FF", "#FFFF00", "#FF00FF", "#00FFFF", "#FFFFFF", "#000000"]}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"on": {"min_delay": 10, "max_delay": 10000, "max_colors": 1}, "off": {"min_delay": 10, "max_delay": 10000, "max_colors": 0}, "rainbow": {"n1": "Step Rate", "min_delay": 10, "max_delay": 10000, "max_colors": 0}, "colour_cycle": {"n1": "Step Rate", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "transition": {"min_delay": 10, "max_delay": 10000, "max_colors": 10}, "chase": {"n1": "Colour 1 Length", "n2": "Colour 2 Length", "n3": "Step 1", "n4": "Step 2", "min_delay": 10, "max_delay": 10000, "max_colors": 2}, "pulse": {"n1": "Attack", "n2": "Hold", "n3": "Decay", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "circle": {"n1": "Head Rate", "n2": "Max Length", "n3": "Tail Rate", "n4": "Min Length", "min_delay": 10, "max_delay": 10000, "max_colors": 2}, "blink": {"min_delay": 10, "max_delay": 10000, "max_colors": 10}, "flicker": {"n1": "Min brightness", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "flame": {"n1": "Min brightness", "n2": "Breath period (ms)", "n3": "Spark gap min (ms, 0=default 10\u201330 s, -1=off)", "n4": "Spark gap max (ms)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "twinkle": {"n1": "Twinkle activity (1\u2013255, higher = more changes)", "n2": "Density (0\u2013255, higher = more of the strip lit)", "n3": "Min adjacent LEDs per twinkle (same as max for fixed length)", "n4": "Max adjacent LEDs per twinkle (same as min for fixed length)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "radiate": {"n1": "Node spacing (LEDs)", "n2": "Out time (ms)", "n3": "In time (ms)", "min_delay": 10, "max_delay": 10000, "max_colors": 2}, "meteor_rain": {"n1": "Tail length", "n2": "Speed (LEDs per frame)", "n3": "Fade amount (1-255)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "scanner": {"n1": "Eye width", "n2": "End pause (frames)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "gradient_scroll": {"n1": "Scroll step rate", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "comet_dual": {"n1": "Tail length", "n2": "Speed", "n3": "Gap", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "sparkle_trail": {"n1": "Spark density", "n2": "Decay", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "wave": {"n1": "Wavelength", "n2": "Amplitude", "n3": "Drift speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "plasma": {"n1": "Scale", "n2": "Speed", "n3": "Contrast", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "segment_chase": {"n1": "Segment size", "n2": "Phase step", "n3": "Segment phase offset", "n4": "Gap per segment", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "bar_graph": {"n1": "Level percent", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "breathing_dual": {"n1": "Phase offset", "n2": "Ease", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "strobe_burst": {"n1": "Burst count", "n2": "Burst gap", "n3": "Cooldown", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "rain_drops": {"n1": "Drop rate", "n2": "Ripple width", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "fireflies": {"n1": "Count", "n2": "Twinkle speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "clock_sweep": {"n1": "Hand width", "n2": "Marker interval", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "marquee": {"n1": "On length", "n2": "Off length", "n3": "Step", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "aurora": {"n1": "Band count", "n2": "Shimmer", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "snowfall": {"n1": "Flake density", "n2": "Fall speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "heartbeat": {"n1": "Pulse 1 ms", "n2": "Pulse 2 ms", "n3": "Pause ms", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "orbit": {"n1": "Orbit count", "n2": "Base speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "palette_morph": {"n1": "Morph ms", "n2": "Warp rate", "n3": "Turbulence", "max_colors": 10, "min_delay": 10, "max_delay": 10000}}
|
||||
{"on": {"min_delay": 10, "max_delay": 10000, "max_colors": 1}, "off": {"min_delay": 10, "max_delay": 10000, "max_colors": 0}, "rainbow": {"n1": "Step Rate", "min_delay": 10, "max_delay": 10000, "max_colors": 0}, "colour_cycle": {"n1": "Step Rate", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "transition": {"min_delay": 10, "max_delay": 10000, "max_colors": 10}, "chase": {"n1": "Colour 1 Length", "n2": "Colour 2 Length", "n3": "Step 1", "n4": "Step 2", "min_delay": 10, "max_delay": 10000, "max_colors": 2}, "pulse": {"n1": "Attack", "n2": "Hold", "n3": "Decay", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "circle": {"n1": "Head Rate", "n2": "Max Length", "n3": "Tail Rate", "n4": "Min Length", "min_delay": 10, "max_delay": 10000, "max_colors": 2}, "blink": {"min_delay": 10, "max_delay": 10000, "max_colors": 10}, "flicker": {"n1": "Min brightness", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "flame": {"n1": "Min brightness", "n2": "Breath period (ms)", "n3": "Spark gap min (ms, 0=default 10\u201330 s, -1=off)", "n4": "Spark gap max (ms)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "twinkle": {"n1": "Twinkle activity (1\u2013255, higher = more changes)", "n2": "Density (0\u2013255, higher = more of the strip lit)", "n3": "Min adjacent LEDs per twinkle (same as max for fixed length)", "n4": "Max adjacent LEDs per twinkle (same as min for fixed length)", "min_delay": 84, "max_delay": 10000, "max_colors": 10}, "radiate": {"n1": "Node spacing (LEDs)", "n2": "Out time (ms)", "n3": "In time (ms)", "min_delay": 10, "max_delay": 10000, "max_colors": 2}, "meteor_rain": {"n1": "Tail length", "n2": "Speed (LEDs per frame)", "n3": "Fade amount (1-255)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "scanner": {"n1": "Eye width", "n2": "End pause (frames)", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "gradient_scroll": {"n1": "Scroll step rate", "min_delay": 10, "max_delay": 10000, "max_colors": 10}, "comet_dual": {"n1": "Tail length", "n2": "Speed", "n3": "Gap", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "sparkle_trail": {"n1": "Spark density", "n2": "Decay", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "wave": {"n1": "Wavelength", "n2": "Amplitude", "n3": "Drift speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "plasma": {"n1": "Scale", "n2": "Speed", "n3": "Contrast", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "segment_chase": {"n1": "Segment size", "n2": "Phase step", "n3": "Segment phase offset", "n4": "Gap per segment", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "bar_graph": {"n1": "Level percent", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "breathing_dual": {"n1": "Phase offset", "n2": "Ease", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "strobe_burst": {"n1": "Burst count", "n2": "Burst gap", "n3": "Cooldown", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "rain_drops": {"n1": "Drop rate", "n2": "Ripple width", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "fireflies": {"n1": "Count", "n2": "Twinkle speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "clock_sweep": {"n1": "Hand width", "n2": "Marker interval", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "marquee": {"n1": "On length", "n2": "Off length", "n3": "Step", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "aurora": {"n1": "Band count", "n2": "Shimmer", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "snowfall": {"n1": "Flake density", "n2": "Fall speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "heartbeat": {"n1": "Pulse 1 ms", "n2": "Pulse 2 ms", "n3": "Pause ms", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "orbit": {"n1": "Orbit count", "n2": "Base speed", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "palette_morph": {"n1": "Morph ms", "n2": "Warp rate", "n3": "Turbulence", "max_colors": 10, "min_delay": 10, "max_delay": 10000}, "gradient_wipe": {"n1": "Start LED", "n2": "End LED", "min_delay": 10, "max_delay": 10000, "max_colors": 2}}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"1": {"name": "on", "pattern": "on", "colors": ["#FFFFFF"], "brightness": 255, "delay": 100, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "2": {"name": "off", "pattern": "off", "colors": [], "brightness": 0, "delay": 100, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "3": {"name": "rainbow", "pattern": "rainbow", "colors": [], "brightness": 255, "delay": 100, "auto": true, "n1": 2, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "4": {"name": "transition", "pattern": "transition", "colors": ["#FF0000", "#00FF00", "#0000FF", "#FFFFFF", "#0000FF", "#FFFF00"], "brightness": 255, "delay": 5000, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1", "palette_refs": [null, null, null, 6, 2, 3]}, "5": {"name": "chase", "pattern": "chase", "colors": ["#FF0000", "#0000FF"], "brightness": 255, "delay": 200, "auto": true, "n1": 5, "n2": 5, "n3": 1, "n4": 1, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "6": {"name": "pulse", "pattern": "pulse", "colors": ["#00FF00"], "brightness": 255, "delay": 500, "auto": true, "n1": 1000, "n2": 500, "n3": 1000, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "7": {"name": "circle", "pattern": "circle", "colors": ["#FFA500", "#800080"], "brightness": 255, "delay": 200, "auto": true, "n1": 2, "n2": 10, "n3": 2, "n4": 5, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "8": {"name": "blink", "pattern": "blink", "colors": ["#FF0000", "#00FF00", "#0000FF", "#FFFF00"], "brightness": 255, "delay": 1000, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "9": {"name": "warm white", "pattern": "on", "colors": ["#FFF5E6"], "brightness": 200, "delay": 100, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "10": {"name": "cool white", "pattern": "on", "colors": ["#E6F2FF"], "brightness": 200, "delay": 100, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "11": {"name": "red", "pattern": "on", "colors": ["#FF0000"], "brightness": 255, "delay": 100, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "12": {"name": "blue", "pattern": "on", "colors": ["#0000FF"], "brightness": 255, "delay": 100, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "13": {"name": "rainbow slow", "pattern": "rainbow", "colors": [], "brightness": 255, "delay": 150, "auto": true, "n1": 1, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "14": {"name": "pulse slow", "pattern": "pulse", "colors": ["#FF6600"], "brightness": 255, "delay": 800, "auto": true, "n1": 2000, "n2": 1000, "n3": 2000, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "15": {"name": "blink red green", "pattern": "blink", "colors": ["#FF0000", "#00FF00"], "brightness": 255, "delay": 500, "auto": true, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1"}, "30": {"name": "rainbow slow", "pattern": "rainbow", "colors": [], "brightness": 255, "delay": 150, "auto": true, "n1": 1, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "31": {"name": "DJ Rainbow", "pattern": "rainbow", "colors": [], "brightness": 220, "delay": 60, "n1": 12, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "32": {"name": "DJ Single Color", "pattern": "on", "colors": ["#ff00ff"], "brightness": 220, "delay": 100, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "33": {"name": "DJ Transition", "pattern": "transition", "colors": ["#ff0000", "#00ff00", "#0000ff"], "brightness": 220, "delay": 250, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "34": {"name": "DJ Rainbow", "pattern": "rainbow", "colors": [], "brightness": 220, "delay": 60, "n1": 12, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "35": {"name": "DJ Single Color", "pattern": "on", "colors": ["#ff00ff"], "brightness": 220, "delay": 100, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "36": {"name": "DJ Transition", "pattern": "transition", "colors": ["#ff0000", "#00ff00", "#0000ff"], "brightness": 220, "delay": 250, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "2"}, "37": {"name": "tranistion2", "pattern": "transition", "colors": ["#FF0000", "#FFFF00", "#FF00FF"], "brightness": 128, "delay": 1000, "n1": 0, "n2": 0, "n3": 0, "n4": 0, "n5": 0, "n6": 0, "n7": 0, "n8": 0, "profile_id": "1", "palette_refs": [0, 3, 4]}, "38": {"name": "Colour Cycle", "pattern": "cLine truncated
|
||||
{"1":{"name":"on","pattern":"on","colors":["#FFFFFF"],"brightness":255,"delay":100,"n1":0,"n2":0,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"1"},"2":{"name":"off","pattern":"off","colors":[],"brightness":0,"delay":100,"n1":0,"n2":0,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"2"},"3":{"name":"rainbow","pattern":"rainbow","colors":[],"brightness":255,"delay":100,"n1":2,"n2":0,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"3"},"4":{"name":"Colour Cycle","pattern":"colour_cycle","colors":["#FF0000","#00FF00","#0000FF"],"brightness":255,"delay":100,"n1":1,"n2":0,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"4"},"5":{"name":"transition","pattern":"transition","colors":["#FF0000","#00FF00","#0000FF"],"brightness":255,"delay":500,"n1":0,"n2":0,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"5"},"6":{"name":"flicker","pattern":"flicker","colors":["#FFB84D"],"brightness":255,"delay":80,"n1":30,"n2":0,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"6"},"7":{"name":"flame","pattern":"flame","colors":[],"brightness":255,"delay":50,"n1":35,"n2":2600,"n3":0,"n4":0,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"7"},"8":{"name":"twinkle","pattern":"twinkle","colors":["#78C8FF","#508CFF","#B478FF","#64DCE8","#A0C8FF"],"brightness":255,"delay":55,"n1":72,"n2":140,"n3":2,"n4":6,"n5":0,"n6":0,"n7":0,"n8":0,"profile_id":"1","auto":true,"id":"8"}}
|
||||
Binary file not shown.
@@ -1,3 +0,0 @@
|
||||
PRST1xœ%ÎÁ
|
||||
Â0Ð_‘ñšCSµJîæ'D$¶«␍ÄÝ’¦ˆˆÿntOovæ²opxz‘´zޱ¦P
|
||||
e®µp={I’g¸ÖÇηÞãlpÍñ>¦¹mÓ”ÂÎþrXŠÀ•¼�[¸Zq«l”²S:e¯þLYn1Ñ%õ‹Ï ¥/ï
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xњ%ОAВ …б»<·,J5\Е4␍К $84SX4Ж»‹eхеНlюШЅ B
|
||||
‹+…„ЫОЬж#§,+м␍‡@ЖЏIб.сщ*Lk{ЊЖ(xJnѓХГ аjЙ°E*)°†m';ЗО©c:зОҐsЭY$‡�hЋѕuиЅ*Qлљ…В?†kJУчП=5Я
|
||||
@@ -1 +0,0 @@
|
||||
PRST1xœ%ÎA л|·, ŠÐK˜ÆP;* ␍……1Þ݉3›—ÿÿf>àø"Ì¹ÖØU–¢°Ä[É¥nì¬Ü8b2˜kz<Ó&Ãà½ÁB9¾œµ±·‚Ðj'vRñ 앃╣rRÎÖZî)Ó5-ò‡Ã÷yÒ-€
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xœEÎÁ
|
||||
Â0Ð_‘9ç`«Qɯˆ”Ô®ˆ»e“RDüwsðô˜™Ë¼ÁñIИx”uS²¬p˜c¤ü¬»J-ç‹Ã¨éþ¨LÅrï½ÃD9¾:¿uˆK„ª9pg¥Ñ#ØÂ»Æ¾á‡Æ±qú1«ÜR¦!Mö¡Ãç��1
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xœ=ÎÝ
|
||||
!†á[‰¯StK[¼€½‰ˆ°v*ÁTü!"º÷Ü¤Žžá�9˜¼¹4bu™VÙ…¢)…’ÿåVÎÁ…”¡÷XO“RœãÀpJöz+žr[R2ÌäÌzäœÁÔ KªÄàE;àKõ´èÓæß¶Ð²£:»Îø%¦p±ŽŽvn? ¼?�¨2ú
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœUÎÁ
|
||||
Â0ЙsM5Uò+"²µ«â¦lSDÄwiNž³3‡ý@èɈPJ2–fª•Uþn×’‹.ˆ§³Ã¨éþ¨Â‹å>‡‰3½}×9ÐZbÕ•ÄÛÀè‘]cß�¡qh7f-·”ù’&ûÁãûF9/.
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xœEÎÁ
|
||||
Â0Ð_‘9ç`«Qɯˆ”Ô®ˆ»e“RDüwsðô˜™Ë¼ÁñIИx”uS²¬p˜c¤ü¬»J-ç‹Ã¨éþ¨LÅrï½ÃD9¾:¿uˆK„ª9pg¥Ñ#ØÂ»Æ¾á‡Æ±qú1«ÜR¦!Mö¡Çç�“1
|
||||
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœ%ͽÂ0àW©Ž5C~•&VÆ
|
||||
¡@�)uª4K…xwR<}ç»Á° —ks<DjÎ)¦…É•B™ë–¸ž¯µža;l¼×Ú{Üž9ïÂ4×ÁÐStl«kævÅ[a'ì…ƒpN¦œ|ˆô}ýmðý‡-‰
|
||||
@@ -1 +0,0 @@
|
||||
PRST1xœMÎ1!†á¿b¾[=5ÌNÎnÆô@I°\€Åÿ»Å.²<oÚ¼Aîéa±?,ŽÅQ�-f‚ÂìZó…xÓþÇ·œr©°'!h~<´î-Õg…k‰÷G#_ùØ0ùä^Ü#7-a;FX ka6ÂVØý˜K1ùKœø_Ÿ/ÐM4y
|
||||
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœ%ͽÂ0àW©Ž5C~•&VÆ
|
||||
¡@�)uª4K…xwR<}ç»Á° —ks<DjÎ)¦…É•B™ë–¸ž¯µža;l¼×Ú{Üž9ïÂ4×ÁÐStl«kævÅ[a'ì…ƒpN¦œ|ˆô}ýmðý‡-‰
|
||||
@@ -1 +0,0 @@
|
||||
PRST1xœMÎ1!†á¿b¾[=5ÌNÎnÆô@I°\€Åÿ»Å.²<oÚ¼Aîéa±?,ŽÅQ�-f‚ÂìZó…xÓþÇ·œr©°'!h~<´î-Õg…k‰÷G#_ùØ0ùä^Ü#7-a;FX ka6ÂVØý˜K1ùKœø_Ÿ/ÐM4y
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,3 +0,0 @@
|
||||
PRST1xœUÎÁ‚0„áw¯=¤jú*†�
|
||||
[m\[²”ƒ1¾»…ž<}ÉÌåÿ ºÁÂsŸ$P˜]Î$ño'Y`¯88ÒÚ{ô
|
||||
7 ÷GŽ´”£5Fa"voX£Üšl–•bÛè2ÆvãXé*¦rªœ+—�Y’LC˜JM³·1•ºAÈo5qeî¿?ªð9±
|
||||
Binary file not shown.
@@ -1,4 +0,0 @@
|
||||
PRST1xśMÎÁ‚0„áwŻ=$ű*†�
|
||||
[%Y[RÚ1ľ»…^<}ÉĚĺ˙Ŕ™7�`ĺPa51rpËäŇ␍tÇĹÚ©×�Â#,ĎWtĽĺŁŞ{…™Ĺě V+�=(†Ä
|
||||
®5m¶ŐťÎŻk@×B[č
|
||||
·B˛oáq™sMu¶ çş1°=z\ľ?µ¦8>
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xśmŹÁ‚0†ßĄ\wČ`ŮMQ^Â2ĄčâÜČ1ĆřînĚ‹‰—~í—?Mű#ďüC™›F 0IďŃ™w¶ÚşÄ˛š7Ľm�ËĺMęveýuUąo<v[şć:'§.Wop
|
||||
ƨĺDN)ąx» �<H¤)B2r"˘Śá@–Ć*ˇNŕ+&gGĄ±WC8<_ßĐéŽńpłhMţ”îýŹ!I°
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xњUЋ;В0птТєp>°WAQґђ5X2Nд8BЬ;©hv¤·SМЃ_BдЙq(,њ’Др·Эg?ЗtEЕЅЦЦжТZіf
|
||||
·иПdНJcЊВ$ћЯ “ЮТJq…PѓЪј…t)ПР‚є]ЁАињњw,q¶ОЛи¦\Wп^rнЕ–є°yЇКѕ?Эh>Ў
|
||||
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœEÎM
|
||||
Â0à«Ès›Eÿ¢’ôE$¶£â¤$Ó…ˆww0góÁ{o1o°„ŠìÊì™)Ã`õ"”Y‹6§˜r�›°ÇFgƒk÷‡0-:k␍Šþ×7~“'y#ná¥Ó™ÒÃ逇Š*ÇÊéÇšÓ-Dº„Eiñù¹L2‡
|
||||
@@ -1,3 +0,0 @@
|
||||
PRST1xœ=ŽA‚0E¯B>Û.
|
||||
*š€KC*ŒØ¤¶¤Æxw�Í{™7‹y!ØÁ€)s5';9
|
||||
\å1Eï¡°XfJA~mø·1ú˜2ÌußkÙÕZo^ls\®ÉÍw”å¸mµÂDÞ>a:Q»r„á´’Bh¤Z)aW°/8tÇ‚ÓKŠ7çip“üÙàý)<¡
|
||||
@@ -1,3 +0,0 @@
|
||||
PRST1xś-ÎÁ‚0Đ_!õ‡Šdo˝ô'Ś!Ş’”–”ĺ`Ś˙î<˝ÍĚö�čfű•‹!Íž‹qs
|
||||
‹cö9J·Çý?RHy]QZkŚÖ’•Zc-n
|
||||
÷<=_ý*“Zk…Ń÷µrşŤ�óćbę„T ␍¨N ŮĹóA{p9čv–śSđý4Ęcľ?6Đ
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1x�5־A‚0…ב«�ַ¶@Dׂ- —0ֶT©�X[2ֶxwG׳ש&»˜‚yXh°M\₪�׀��‚ֹ8…�0[
|
||||
’ור/חט#%ט=ֺ¾†q”·r\…¹כ�ƒMע¥©*…ֹzף„מd5Gh¦ֵ*„Zz+6b-1l ¿´™m¦ֻל2ֺLסגה"7ֹy5�־ד:G
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xœ-ÎÁ Ð_1ã•ÔZŽúÆ´«’ 4°Õã¿»Š§7;sÙ¢»,˜
|
||||
/îNP˜3å(í¿8¥�r�Ýa©õ¶ìŽÙ_®©ÈÐh0RpOØN¢›9ÁržI!XÓˆ�ØËW„ö{+]eSéL9�} ƒåƒ÷ªù0¿
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1x�=ЮA
|
||||
Т0аЋШw�EZ5JаK�б�ZH��L"онС�Ћ7ќџѓFЄ�с!\e�е�`�I�KдќнRHЅТ�и�ЕЮсlp-ѓу)�ЋНЕzС;=i�/ee�иiІє:Sv�=МютЁсЧЦщG.щ>ОЬ�Овсѓ,�1И
|
||||
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœ5ÎA‚0Ы�϶‹‚ˆ¦è%Œ!F �–´ÃÂïîhu6o2ÿ/æ ïV‚Sâ"Ѹ’碟\"(lŽ™¢—ø—tÿ¤Kˆ æ‚ÒZ-#·ò£µ¸*Üâ<Nì)I¥ÖZa Å=`ZYÝΆãN
|
||||
¾‚i„¦0RðMæ˜i3§ÌùËÃ}^¨›ùÂë␍Çü8÷
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœ5Î=Â0†á«Tk†þQ�À%*T%Ô@¥’TŽ; ÄÝIáå±ôzðÞ¾å¨ET Ž·JT,V•ŧšÃð·0‰ ‡Ë>¸8™OõS¨ËÒ`äÙ¾A]Zíª¤²²�¯@M¢ÎÉ7 v;÷-hã˜é2§Ìyg‘pŸf¦1ýTáû^
|
||||
7˜
|
||||
@@ -1,3 +0,0 @@
|
||||
PRST1x�5ΞΝ
|
||||
Β0ΰW)γ5‡ώhΉϊ"%ΪU5)›νAΔww5xϊ–™9μΑ=BI
|
||||
v>Η%Α`q"ΔA»o<ώγK�#'Ψ#6‡²ο†'ƒ3ϋΫ]%-κ²4�hvOΨVO·J„^Ι T°MΦ��ΐκ"l3»L›ΩgΊΗ«�iτ“ώSαύ��5%
|
||||
@@ -1,4 +0,0 @@
|
||||
PRST1xœMαÂ0Ð_A×5CZ ´™Q~!¨‘BR%î€ÿŽE¦gÝÝà7¢{˜
|
||||
ofŸiž
|
||||
ÇL9JõŸÞRH¹ÀœÐX{Ô½–¬µµ£ÆYášýýÁ‘ŠL:&
|
||||
îÓËéVN0œWRˆdB3[Ä]e_é+‡ÊðcÉiö�.~’¿Z|¾¡ 61
|
||||
@@ -1 +0,0 @@
|
||||
PRST1xœ5ŽAƒ E¯b¾[¨U+WiŒ¡2¶¦`š¦éÝ’nxÌ›Y¼Œ|ùPÌÚÎ�¿ˆ60l2r&.?ýýlµuâ‚Rõ|àCt%Wuß5®n½Ýƒ!OjÎiùN¹ÜN¦‚¨¢35DÑ@¤é”Ñft}ÆùÀæì²jšVÓª#TSL�-)ËìZ³ôŒßQ•AÓ
|
||||
@@ -1 +0,0 @@
|
||||
PRST1xњEО1В0Р« ПљЎiЎ ЂK „5)MЪФвоXНЂ—gщяБD72В‹lF—зВѓЙ‰pЋьoчR^@glOлаbpЛющ’И‹mУЬФлкЉ$ђдВС‚:ҐХљТЃ¬Іi/о+}еP9®L9=|а«ф‹пжg2д
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xœ=ÎÍ
|
||||
Â0àW‘é5‡ô?ìM"} ‰vÕBMJ’D|wSž¾afû†5O!rˆ;³zç␍#{›úuu³ótDQi”N]ÑÊ^␍Öö²•�“ÀÅO÷G´Òq%¥Àȳy�ºÍ(ú•l ª¨I“AM¦Ít™>£~,Þݦ™ÏÓ˜þ+ñùÚ¦5ó
|
||||
@@ -1,3 +0,0 @@
|
||||
PRST1x°Mна
|
||||
б0ЮW▒вз╘SzTЯ%D╓╨L╣m├┬ЬНfКе\╬ДOЫ ╦'а┌)С"┤ЬЙ°ВP3╔ ⌡©П}LЖ└Й8≈dуNЖр²╝╘©?8P√⌠Zk┘√╪{ц6р╨▒#,╖▒┌≥Жb␍k└%Л4╜
|
||||
КBSь╤3C┼]ОИр╥Р≈аВ┘┘6
|
||||
@@ -1,2 +0,0 @@
|
||||
PRST1xœMÎK
|
||||
Â0…á½§ÜT£’tR$Ú«âMÉc âÞm�ˆ£þ39Oˆ»3,¦2Car¥p’¿rŽ!¦{ÀЍï‰0(œ’¿ÞŠpž‡Î…‘ƒ{À"WK„-©²‚hXMK•î;Ëú—6°¦±mìûSŠøèÇù’Æë␍»0Ø
|
||||
@@ -1,4 +0,0 @@
|
||||
PRST1xœMÎA‚0Ы˜ï¶‹RÉ€KcŠŒBR[Òc¼»l\½Éÿùɼáí“ANr˜ÙFÙ
|
||||
V+ÂÑçê?½b␍8ö½éj�‹Â—Ç,žS.ŒÖ
|
||||
;ûµù´›�Ä�|ªL½uŨ)_ƒ
|
||||
ÍN»sÞé~¬1ÜÇ×eÊ/Uø|Æœ5
|
||||
Binary file not shown.
@@ -1,3 +0,0 @@
|
||||
PRST1xœ5ŽA‚0E¯B>Û.
|
||||
*š€KCªŒBRÚ¦c¼»ÅÙ¼7óÿb>ðv"0Í\D눙Š)¤8@!ZÙ’—xOºò.¤æŠ²mµŒÜJW϶:n
|
||||
÷4¾ö4K¹ÖZ¡'gß0�¨]8ÀpZHÁW0ÕVðõÞô˜ÇŒSF“qθlˆ)<GGÝØË«¾?ð¹<
|
||||
@@ -1,3 +0,0 @@
|
||||
PRST1xœMŽ1Â0Eïò»fp
|
||||
�(K/�
|
||||
�H!©Òt@ˆ»cÈÂô¾Ÿ¿%¿�üƒá0†2F†Âìkå’þÕ˜c.ÜÝ0‘¸Î‘%œ.%Üî5ñ"•Þ…‰£J&RðkÍpµ¬¬�´HA§e•6mÜÂÉQ2p_¹kØ7Øæ’¯!ò9Lò–Æû¼Ã1ó
|
||||
Binary file not shown.
@@ -1,2 +0,0 @@
|
||||
PRST1xœ%ÎK
|
||||
Ã0Ы”éÖ‹$ýâ«”ÜFnŽ›PJï^ÇÖæI£Í|Áf&hlFæÃ6¹HPXLŒ$œãÀù|d…~àhË WxŠ{O‘iÍ�®iFòæÝî»I1@GI¤À-tޏ«œ*çÊ¥rÜ*÷Â"Á:Oƒs�¶´ò”{␍Bv/ÃÉûþ÷½8ù
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"1": {"name": "default", "type": "zones", "zones": ["1", "8"], "scenes": [], "palette_id": "1"}, "2": {"name": "test", "type": "zones", "zones": ["6", "7"], "scenes": [], "palette_id": "12"}}
|
||||
{"1": {"name": "default", "type": "zones", "zones": ["1"], "scenes": [], "palette_id": "1", "id": "1"}}
|
||||
+1
-22
@@ -1,22 +1 @@
|
||||
{
|
||||
"1": {
|
||||
"name": "Default Scene",
|
||||
"sequences": [
|
||||
"1"
|
||||
],
|
||||
"groups": [
|
||||
"1"
|
||||
]
|
||||
},
|
||||
"2": {
|
||||
"name": "Party Mode",
|
||||
"sequences": [
|
||||
"1",
|
||||
"2"
|
||||
],
|
||||
"groups": [
|
||||
"1",
|
||||
"2"
|
||||
]
|
||||
}
|
||||
}
|
||||
{}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"1": {"name": "Main Group", "zone_id": null, "loop": true, "repeat_count": 0, "steps": [{"preset_id": "1", "duration_s": 3.0}, {"preset_id": "2", "duration_s": 3.0}], "active": false}, "2": {"name": "Accent Group", "zone_id": null, "loop": true, "repeat_count": 0, "steps": [{"preset_id": "2", "duration_s": 2.0}, {"preset_id": "3", "duration_s": 2.0}], "active": false}}
|
||||
{}
|
||||
+1
-1
@@ -1 +1 @@
|
||||
{"1": {"name": "default", "names": ["led-188b0e1560a8", "led-f0f5bdfb9d30"], "presets": [["4", "2", "7"], ["3", "14", "5"], ["8", "9", "1"], ["6", "38", "42"], ["39", "40", "41"], ["43", "44", "45"], ["46", "47", "48"], ["49", "50", "51"], ["52", "53", "54"], ["55", "56", "57"], ["58", "59", "60"], ["61", "62"]], "presets_flat": ["4", "2", "7", "3", "14", "5", "8", "9", "1", "6", "38", "42", "39", "40", "41", "43", "44", "45", "46", "47", "48", "49", "50", "51", "52", "53", "54", "55", "56", "57", "58", "59", "60", "61", "62"], "default_preset": "41"}, "2": {"name": "default", "names": ["1", "2", "3", "4", "5", "6", "7", "8", "0", "a"], "presets": [["16", "17", "18"], ["19", "20", "21"], ["22", "23", "24"], ["25", "26", "27"], ["28", "29", "30"]], "presets_flat": ["16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30"]}, "3": {"name": "default", "names": ["1"], "presets": [], "default_preset": null}, "4": {"name": "default", "names": ["1"], "presets": [], "default_preset": null}, "5": {"name": "dj", "names": ["dj"], "presets": [["31", "32", "33"]], "default_preset": "31", "presets_flat": ["31", "32", "33"]}, "6": {"name": "default", "names": ["1"], "presets": [], "default_preset": null, "presets_flat": []}, "7": {"name": "dj", "names": ["dj"], "presets": [["34", "35", "36"]], "default_preset": "34", "presets_flat": ["34", "35", "36"]}, "8": {"name": "test", "names": ["led-188b0e1560a8"], "presets": [["1", "2", "3"], ["4", "5"]], "default_preset": "1", "presets_flat": ["1", "2", "3", "4", "5"]}}
|
||||
{"1": {"name": "default", "names": ["1"], "presets": [["1", "2", "3", "4", "5", "6", "7", "8"]], "presets_flat": ["1", "2", "3", "4", "5", "6", "7", "8"], "default_preset": "1", "id": "1", "group_ids": ["1"], "preset_group_ids": {}, "sequence_ids": [], "content_kind": "presets"}}
|
||||
@@ -0,0 +1,67 @@
|
||||
dependencies:
|
||||
espressif/cjson:
|
||||
component_hash: e788323270d90738662d66fffa910bfe1fba019bba087f01557e70c40485b469
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 1.7.19~2
|
||||
espressif/mdns:
|
||||
component_hash: ef0766474b37459f6ad22461e453a8713e780e831a7c993b16919d817988dff6
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 1.14.1
|
||||
espressif/w5500:
|
||||
component_hash: f76a95ce7a6029cf93aa39d47067588b24263852031f11c71e5d192ad915ec07
|
||||
dependencies:
|
||||
- name: espressif/wiznet_common
|
||||
registry_url: https://components.espressif.com
|
||||
require: private
|
||||
version: ^1.1.0
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=6.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 2.0.1
|
||||
espressif/wiznet_common:
|
||||
component_hash: c32814cb872f0dca09f0c97ab231f9264a65bacc13fab6baf65f71959269a73d
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.3'
|
||||
source:
|
||||
registry_url: https://components.espressif.com
|
||||
type: service
|
||||
version: 1.1.0
|
||||
idf:
|
||||
source:
|
||||
type: idf
|
||||
version: 6.1.0
|
||||
joltwallet/littlefs:
|
||||
component_hash: e1a0f4a6adf821759cb34703a91107bbdce5d7ff6f4728f40ab012bb87842974
|
||||
dependencies:
|
||||
- name: idf
|
||||
require: private
|
||||
version: '>=5.0'
|
||||
source:
|
||||
registry_url: https://components.espressif.com/
|
||||
type: service
|
||||
version: 1.23.1
|
||||
direct_dependencies:
|
||||
- espressif/cjson
|
||||
- espressif/mdns
|
||||
- espressif/w5500
|
||||
- joltwallet/littlefs
|
||||
manifest_hash: d66adeab020ae08629bcb8999b623eb65117c51257a671ab53bd5dc8e8045975
|
||||
target: esp32s3
|
||||
version: 3.0.0
|
||||
@@ -0,0 +1,12 @@
|
||||
{$DOMAIN} {
|
||||
encode zstd gzip
|
||||
reverse_proxy controller:8080
|
||||
request_body {
|
||||
max_size 20MB
|
||||
}
|
||||
header {
|
||||
Strict-Transport-Security "max-age=31536000"
|
||||
X-Content-Type-Options "nosniff"
|
||||
Referrer-Policy "same-origin"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
# led-controller for the desktop: the firmware's sources built with a plain C compiler, no ESP-IDF.
|
||||
# Build from the repository root with `make desktop`, or here:
|
||||
# cmake -S desktop -B build-desktop && cmake --build build-desktop
|
||||
# Needs www_dist/www.pack (tools/build_web.sh). Third-party code is vendored: desktop/tools/vendor.sh.
|
||||
cmake_minimum_required(VERSION 3.16)
|
||||
project(led_controller_desktop VERSION 0.1.0 LANGUAGES C)
|
||||
|
||||
set(CMAKE_C_STANDARD 11)
|
||||
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||
set(CMAKE_C_EXTENSIONS ON)
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
set(ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..)
|
||||
set(WWW_PACK ${ROOT}/www_dist/www.pack)
|
||||
if(NOT EXISTS ${WWW_PACK})
|
||||
message(FATAL_ERROR "www_dist/www.pack is missing: run tools/build_web.sh (make web)")
|
||||
endif()
|
||||
# The simulated rig at /sim/ (tools/build_sim_web.sh). Optional: an empty pack when it is not built.
|
||||
set(SIM_PACK ${ROOT}/www_dist/sim.pack)
|
||||
if(NOT EXISTS ${SIM_PACK})
|
||||
file(WRITE ${SIM_PACK} "WWW1")
|
||||
endif()
|
||||
|
||||
execute_process(COMMAND git -C ${ROOT} describe --tags --always --dirty
|
||||
OUTPUT_VARIABLE LED_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
|
||||
if(NOT LED_VERSION)
|
||||
set(LED_VERSION "desktop")
|
||||
endif()
|
||||
|
||||
# The firmware's app sources (the same list as main/CMakeLists.txt's linux branch, plus the bridge transport)
|
||||
set(MAIN_SRC
|
||||
beat.c beat_dsp.c espnow_tx.c bridge.c store.c api.c web.c envelope.c engine.c rel.c bench.c audio.c
|
||||
res_profile.c res_misc.c res_device.c res_group.c seqplay.c res_bundle.c audio_api.c patterns.c outputs.c
|
||||
ws.c ws_client.c wifi_drv.c bridge_serial.c auth.c)
|
||||
list(TRANSFORM MAIN_SRC PREPEND ${ROOT}/main/)
|
||||
|
||||
# The host stand-ins for the board's hardware (mic, OTA, littlefs, timers) and the app_main that starts it
|
||||
set(SIM_SRC sim_main.c sim_stubs.c sim_timer.c sim_rig.c)
|
||||
list(TRANSFORM SIM_SRC PREPEND ${ROOT}/sim-controller/components/sim/)
|
||||
|
||||
set(THIRD_PARTY
|
||||
third_party/esp_http_server/src/httpd_main.c
|
||||
third_party/esp_http_server/src/httpd_parse.c
|
||||
third_party/esp_http_server/src/httpd_sess.c
|
||||
third_party/esp_http_server/src/httpd_txrx.c
|
||||
third_party/esp_http_server/src/httpd_uri.c
|
||||
third_party/esp_http_server/src/httpd_ws.c
|
||||
third_party/esp_http_server/src/util/ctrl_sock.c
|
||||
third_party/http_parser/http_parser.c
|
||||
third_party/cjson/cJSON.c)
|
||||
|
||||
set(PORT
|
||||
port/src/freertos.c
|
||||
port/src/esp_glue.c
|
||||
port/src/desktop_crypto.c
|
||||
port/src/www_pack.c
|
||||
port/src/sim_pack.c
|
||||
port/src/mdns_host.c
|
||||
port/src/mic_alsa.c)
|
||||
|
||||
add_executable(led-controller desktop_main.c ${MAIN_SRC} ${SIM_SRC} ${THIRD_PARTY} ${PORT})
|
||||
# www_pack.c embeds the pack with .incbin, which CMake can't see: rebuild it when the pack changes
|
||||
set_source_files_properties(port/src/www_pack.c PROPERTIES OBJECT_DEPENDS ${WWW_PACK})
|
||||
set_source_files_properties(port/src/sim_pack.c PROPERTIES OBJECT_DEPENDS ${SIM_PACK})
|
||||
target_include_directories(led-controller PRIVATE
|
||||
port/include
|
||||
${ROOT}/main
|
||||
${ROOT}/bridge/main
|
||||
${ROOT}/sim-controller/components/sim
|
||||
${ROOT}/sim-controller/components/sim/include
|
||||
third_party/esp_http_server/include
|
||||
third_party/esp_http_server/src
|
||||
third_party/esp_http_server/src/util
|
||||
third_party/http_parser
|
||||
third_party/cjson)
|
||||
target_compile_definitions(led-controller PRIVATE
|
||||
_GNU_SOURCE
|
||||
LED_DESKTOP_MDNS
|
||||
LED_DESKTOP_ALSA
|
||||
LED_APP_VERSION="${LED_VERSION}"
|
||||
WWW_PACK="${WWW_PACK}"
|
||||
SIM_PACK="${SIM_PACK}")
|
||||
# ESP-IDF force-includes the build's sdkconfig.h into every source; the desktop port has the same file
|
||||
target_compile_options(led-controller PRIVATE -include ${CMAKE_CURRENT_SOURCE_DIR}/port/include/sdkconfig.h
|
||||
-Wall -Wno-unused-function -Wno-unused-variable -Wno-unused-parameter)
|
||||
find_library(ASOUND_LIB asound REQUIRED) # ALSA capture for the microphone (libasound2-dev at build time)
|
||||
target_link_libraries(led-controller PRIVATE Threads::Threads m rt ${ASOUND_LIB})
|
||||
install(TARGETS led-controller RUNTIME DESTINATION bin)
|
||||
|
||||
# Headless install: a .deb with the program, a systemd service and its default settings (`make deb`).
|
||||
# The service runs as a dynamic user with its store in /var/lib/led-controller.
|
||||
set(CPACK_PACKAGING_INSTALL_PREFIX /usr)
|
||||
set(CMAKE_INSTALL_PREFIX /usr)
|
||||
install(FILES packaging/led-controller.service DESTINATION lib/systemd/system)
|
||||
install(FILES packaging/led-controller.default DESTINATION /etc/default RENAME led-controller)
|
||||
install(FILES packaging/led-bridge.udev.example DESTINATION share/doc/led-controller)
|
||||
install(FILES ${ROOT}/README.md DESTINATION share/doc/led-controller RENAME README.md OPTIONAL)
|
||||
install(FILES ${ROOT}/bridge/README.md DESTINATION share/doc/led-controller RENAME bridge-README.md OPTIONAL)
|
||||
|
||||
# The command-line tool (led-tool, a PyInstaller binary) and the LED driver firmware image (led-driver, built with
|
||||
# ESP-IDF v6.1). Both are bundled into the .deb when LED_BUNDLE_TOOLS is on (`make deb` turns it on).
|
||||
option(LED_BUNDLE_TOOLS "Install led-cli and the led-driver firmware image into the package" OFF)
|
||||
set(LED_CLI_BIN ${ROOT}/led-tool/dist/led-cli)
|
||||
set(LED_DRIVER_BIN ${ROOT}/led-driver/build-v61/led_driver.bin)
|
||||
if(LED_BUNDLE_TOOLS)
|
||||
if(NOT EXISTS ${LED_CLI_BIN})
|
||||
message(FATAL_ERROR "led-tool/dist/led-cli is missing: run make led-cli")
|
||||
endif()
|
||||
if(NOT EXISTS ${LED_DRIVER_BIN})
|
||||
message(FATAL_ERROR "led-driver/build-v61/led_driver.bin is missing: build the driver with the ESP-IDF v6.1 image")
|
||||
endif()
|
||||
install(PROGRAMS ${LED_CLI_BIN} DESTINATION bin)
|
||||
install(FILES ${LED_DRIVER_BIN} DESTINATION share/led-controller/firmware)
|
||||
endif()
|
||||
|
||||
set(CPACK_PACKAGE_NAME "led-controller")
|
||||
set(CPACK_PACKAGE_VERSION "${PROJECT_VERSION}")
|
||||
set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "LED rig controller: web UI, presets, sequences and ESP-NOW to the LED drivers")
|
||||
set(CPACK_PACKAGE_CONTACT "led-controller packagers <packaging@localhost>")
|
||||
set(CPACK_GENERATOR "DEB")
|
||||
set(CPACK_DEBIAN_PACKAGE_SECTION "misc")
|
||||
set(CPACK_DEBIAN_PACKAGE_PRIORITY "optional")
|
||||
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
|
||||
set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
|
||||
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA ${CMAKE_CURRENT_SOURCE_DIR}/packaging/postinst
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/packaging/prerm
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/packaging/postrm
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/packaging/conffiles)
|
||||
include(CPack)
|
||||
@@ -0,0 +1,69 @@
|
||||
# led-controller on a desktop
|
||||
|
||||
The controller firmware (`main/`) built as a program for Linux: no ESP-IDF, no Docker, no board. The web
|
||||
UI and API are the same as the board's. ESP-NOW goes through a bridge: the USB bridge board (`bridge/`)
|
||||
or a UDP bridge such as `led-simulator`.
|
||||
|
||||
## Build and run
|
||||
|
||||
Needs `cmake`, a C compiler, and `www_dist/www.pack` (`make web` builds it with `node`).
|
||||
|
||||
```bash
|
||||
make desktop # builds build-desktop/led-controller
|
||||
make desktop-run # the web UI on http://localhost:8080/
|
||||
make test-desktop # starts the program and runs the API tests against it
|
||||
```
|
||||
|
||||
Or run it directly:
|
||||
|
||||
```bash
|
||||
build-desktop/led-controller --db ./data --port 8080
|
||||
```
|
||||
|
||||
| option | environment | meaning |
|
||||
| --- | --- | --- |
|
||||
| `--db DIR` | `LED_CONTROLLER_DB` | the store (default `/db`) |
|
||||
| `--port N` | `LED_CONTROLLER_HTTP_PORT` | web UI and API port (default 8080) |
|
||||
|
||||
`LED_LOG_LEVEL=error|warn|info|debug|verbose` sets how much is logged (default `info`).
|
||||
|
||||
## Install as a service (.deb)
|
||||
|
||||
```bash
|
||||
make deb # build-desktop/led-controller_0.1.0_<arch>.deb
|
||||
sudo apt install ./build-desktop/led-controller_0.1.0_*.deb
|
||||
```
|
||||
|
||||
The package installs `/usr/bin/led-controller`, the `led-cli` command-line tool (`/usr/bin/led-cli`), the LED driver
|
||||
firmware image (`/usr/share/led-controller/firmware/led_driver.bin`), a systemd unit (`led-controller.service`) that runs it as a
|
||||
dynamic user with its store in `/var/lib/led-controller`, and `/etc/default/led-controller` for the port. The service starts at install and restarts after a restart from
|
||||
the web UI. The bridges are kept in the controller's settings, so set them in the web UI under Settings > Network >
|
||||
Bridges. Change the port in `/etc/default/led-controller`, then `sudo systemctl restart led-controller`. The bridge's serial port is
|
||||
owned by `dialout`, which the service joins.
|
||||
|
||||
`packaging/led-bridge.udev.example` (installed in `/usr/share/doc/led-controller/`) gives the bridge board a
|
||||
stable name, `/dev/led-bridge`, by its serial number.
|
||||
|
||||
## How it is built
|
||||
|
||||
`desktop/CMakeLists.txt` compiles the firmware's own sources (`main/`, and the host stand-ins in
|
||||
`sim-controller/components/sim/`: mic, OTA, littlefs and the timers) with a small port of the ESP-IDF APIs
|
||||
they use:
|
||||
|
||||
- `port/src/freertos.c`: tasks, queues, semaphores, stream buffers and event groups on POSIX threads.
|
||||
- `port/src/esp_glue.c`: logging, error names, random numbers, the MAC (from the first real network
|
||||
interface), restart (an exit, for the service manager), and the app description.
|
||||
- `port/src/desktop_crypto.c`: SHA-1 and base64 for the WebSocket handshake.
|
||||
- `port/src/www_pack.c`: the web UI pack (`www_dist/www.pack`) embedded in the program.
|
||||
- `port/include/`: the headers. `sdkconfig.h` is the config the sim's build generated.
|
||||
|
||||
Third-party code is vendored in `third_party/`: the HTTP server and `http_parser` from ESP-IDF v6.1, and cJSON.
|
||||
`tools/vendor.sh` refreshes them (it runs `espressif/idf:v6.1` to copy the httpd sources). Its patch
|
||||
(`tools/patch_vendor.py`) swaps the handshake's mbedTLS and PSA calls for `desktop_crypto.c`.
|
||||
|
||||
## Limits
|
||||
|
||||
- ESP-NOW needs the bridge. Without one the program runs, and its ESP-NOW frames go nowhere.
|
||||
- The Wi-Fi drivers are not discovered automatically: their mDNS discovery is not in the desktop build yet.
|
||||
- Firmware OTA is not available (the API answers 501, as in the simulator).
|
||||
- The mic is simulated: beats come from the BPM clock, as in the simulator.
|
||||
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* led-controller as a desktop program: the firmware's app_main (sim-controller/components/sim/sim_main.c)
|
||||
* with its settings from the command line or the environment. No ESP-IDF is needed to build or run it.
|
||||
*
|
||||
* led-controller [--db DIR] [--port N]
|
||||
*
|
||||
* The bridges (the USB bridge board, /dev/ttyACM1, or host:port over UDP such as led-simulator) are kept in
|
||||
* the store's settings: set them in the web UI under Settings > Network > Bridges. The environment works
|
||||
* too: LED_CONTROLLER_DB, LED_CONTROLLER_HTTP_PORT. Without a bridge the program runs, but its ESP-NOW frames
|
||||
* go nowhere.
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
|
||||
void app_main(void);
|
||||
|
||||
static void set_from_env(const char *from, const char *to)
|
||||
{
|
||||
const char *v = getenv(from);
|
||||
if (v && v[0] && !(getenv(to) && getenv(to)[0])) {
|
||||
setenv(to, v, 1);
|
||||
}
|
||||
}
|
||||
|
||||
static void usage(FILE *out)
|
||||
{
|
||||
fprintf(out,
|
||||
"usage: led-controller [--db DIR] [--port N]\n"
|
||||
" --db DIR store directory (default /db)\n"
|
||||
" --port N web UI and API port (default 8080)\n"
|
||||
" bridges are set in the web UI (Settings > Network > Bridges)\n");
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
for (int i = 1; i < argc; i++) {
|
||||
const char *a = argv[i];
|
||||
if (!strcmp(a, "-h") || !strcmp(a, "--help")) {
|
||||
usage(stdout);
|
||||
return 0;
|
||||
}
|
||||
if ((!strcmp(a, "--db") || !strcmp(a, "--port")) && i + 1 < argc) {
|
||||
const char *val = argv[++i];
|
||||
setenv(!strcmp(a, "--db") ? "LED_SIM_DB" : "LED_SIM_HTTP_PORT", val, 1);
|
||||
continue;
|
||||
}
|
||||
fprintf(stderr, "led-controller: unknown option %s\n", a);
|
||||
usage(stderr);
|
||||
return 2;
|
||||
}
|
||||
set_from_env("LED_CONTROLLER_DB", "LED_SIM_DB");
|
||||
set_from_env("LED_CONTROLLER_HTTP_PORT", "LED_SIM_HTTP_PORT");
|
||||
app_main(); // runs until the process is stopped
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
/etc/default/led-controller
|
||||
Loaded 100 of 1473 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user