jason/chorus
1
0
Fork 0
forked from jason/echo
Code Pull Requests Activity

ver 1.2

Browse Source
This commit is contained in:
jason
2026-06-22 23:55:19 -05:00
parent 151b962662
commit d34fbf7626
15 changed files with 571 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
echo-memory-1.2.0.plugin
BIN
View File
Binary file not shown.
echo-memory-performance-brief.html
+222
View File
@@ -0,0 +1,222 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>ECHO Memory — Performance Update</title>
<style>
:root{
--ink:#16181d; --ink-soft:#3a3f4a; --muted:#6b7280;
--line:#e4e7ec; --line-soft:#eef1f4;
--bg:#ffffff; --panel:#f7f8fa; --panel-2:#f0f4f9;
--accent:#2f6df6; --accent-deep:#1b3a8f; --accent-soft:#e8f0ff;
--good:#0f9d6b; --good-soft:#e6f6ef; --bad:#d64545; --bad-soft:#fdecec;
--warn:#b7791f; --code-bg:#0f1320; --code-ink:#e6e9f2;
--radius:14px;
}
*{box-sizing:border-box;}
html{-webkit-print-color-adjust:exact; print-color-adjust:exact;}
body{
margin:0; color:var(--ink); background:#eceef1;
font-family:-apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;
line-height:1.55; font-size:15px;
}
.page{
max-width:880px; margin:32px auto; background:var(--bg);
padding:0 0 56px; border-radius:18px; overflow:hidden;
box-shadow:0 10px 40px rgba(20,24,35,.10);
}
.hero{
background:linear-gradient(135deg,#101a3a 0%,#1b3a8f 55%,#2f6df6 100%);
color:#fff; padding:48px 56px 40px; position:relative;
}
.hero .kicker{font-size:12px; letter-spacing:.18em; text-transform:uppercase; opacity:.82; margin:0 0 14px;}
.hero h1{font-size:38px; line-height:1.08; margin:0 0 10px; font-weight:750; letter-spacing:-.5px;}
.hero .sub{font-size:16.5px; opacity:.92; max-width:640px; margin:0 0 22px;}
.hero .meta{display:flex; flex-wrap:wrap; gap:10px 26px; font-size:13px; opacity:.92; border-top:1px solid rgba(255,255,255,.18); padding-top:16px;}
.hero .meta b{font-weight:650;}
.pill{display:inline-block; background:rgba(255,255,255,.16); border:1px solid rgba(255,255,255,.28); padding:3px 11px; border-radius:999px; font-size:12px; font-weight:600; margin-right:6px;}
.body{padding:8px 56px 0;}
section{padding:28px 0; border-bottom:1px solid var(--line-soft);}
section:last-child{border-bottom:none;}
h2{font-size:13px; letter-spacing:.14em; text-transform:uppercase; color:var(--accent-deep); margin:0 0 4px; font-weight:700;}
.h2sub{font-size:22px; font-weight:700; letter-spacing:-.3px; margin:0 0 16px; color:var(--ink);}
h3{font-size:15.5px; margin:20px 0 7px; font-weight:680;}
p{margin:0 0 13px; color:var(--ink-soft);}
strong{color:var(--ink);}
code{background:var(--panel-2); padding:1px 6px; border-radius:5px; font-family:"SF Mono",ui-monospace,Menlo,Consolas,monospace; font-size:12.5px; color:var(--accent-deep);}
.lede{font-size:16.5px; color:var(--ink); background:var(--accent-soft); border-left:4px solid var(--accent); padding:15px 19px; border-radius:0 10px 10px 0; margin:0;}
.metrics{display:grid; grid-template-columns:repeat(4,1fr); gap:12px; margin:4px 0 6px;}
.metric{background:var(--panel); border:1px solid var(--line); border-radius:var(--radius); padding:16px 13px; text-align:center;}
.metric .num{font-size:30px; font-weight:780; letter-spacing:-1px; line-height:1; color:var(--accent-deep);}
.metric .num.good{color:var(--good);}
.metric .lbl{font-size:11.5px; color:var(--muted); margin-top:7px; line-height:1.35;}
.grid{display:grid; gap:13px;}
.g2{grid-template-columns:repeat(2,1fr);}
.card{background:var(--panel); border:1px solid var(--line); border-radius:var(--radius); padding:15px 16px;}
.card h4{margin:0 0 5px; font-size:14.5px; font-weight:680;}
.card p{margin:0; font-size:13px; color:var(--muted); line-height:1.5;}
table{width:100%; border-collapse:collapse; font-size:13px; margin:8px 0 6px;}
th,td{text-align:left; padding:9px 11px; border-bottom:1px solid var(--line);}
th{background:var(--panel-2); font-size:11px; letter-spacing:.05em; text-transform:uppercase; color:var(--ink-soft); font-weight:680;}
td.n{font-variant-numeric:tabular-nums; text-align:right; white-space:nowrap;}
tr td:first-child{font-weight:600; color:var(--ink);}
.delta{font-weight:700; color:var(--good);}
.muted-row td{color:var(--muted);}
/* bar chart */
.bars{margin:14px 0 4px;}
.bar{display:flex; align-items:center; gap:12px; margin:0 0 9px;}
.bar .name{flex:0 0 168px; font-size:12.5px; color:var(--ink-soft); text-align:right;}
.bar .track{flex:1; background:var(--panel-2); border-radius:7px; height:26px; position:relative; overflow:hidden;}
.bar .fill{height:100%; border-radius:7px; background:linear-gradient(90deg,#2f6df6,#1b3a8f); display:flex; align-items:center; justify-content:flex-end; padding-right:9px; color:#fff; font-size:12px; font-weight:700;}
.bar .fill.old{background:linear-gradient(90deg,#e87f7f,#d64545);}
.bar .fill.best{background:linear-gradient(90deg,#23b783,#0f9d6b);}
.note{background:var(--good-soft); border:1px solid #bfe6d4; border-radius:10px; padding:12px 16px; font-size:12.5px; color:#0c5c43; margin:14px 0 0;}
.warn{background:#fff6e6; border:1px solid #f0d9a8; color:#7a5712;}
.foot{padding:22px 56px 0; color:var(--muted); font-size:11.5px; line-height:1.5;}
.foot b{color:var(--ink-soft);}
@media print{
body{background:#fff;}
.page{margin:0; box-shadow:none; border-radius:0; max-width:none; padding-bottom:24px;}
@page{ size:A4; margin:13mm; }
section,.metrics,.bars,table,.grid{break-inside:avoid;}
.hero{border-radius:0;}
}
</style>
</head>
<body>
<div class="page">
<div class="hero">
<p class="kicker">ECHO Memory · Performance Update</p>
<h1>Full-vault operations: from timing out to sub-second</h1>
<p class="sub">Releases v1.1.0 and v1.2.0 rebuilt the vault's network layer and entity
resolution. Long operations that used to exceed the agent tool timeout and drop the
session now complete in under a second — and shortened entity names resolve instead of
spawning duplicates.</p>
<div class="meta">
<span><span class="pill">v1.1.0</span><span class="pill">v1.2.0</span></span>
<span><b>Prepared</b> 2026-06-22</span>
<span><b>Vault</b> 186 notes · 119 entities</span>
<span><b>Endpoint</b> echoapi.alwisp.com</span>
</div>
</div>
<div class="body">
<section>
<p class="lede">Scripts that sweep the whole vault — <code>sweep.py</code>,
<code>vault_lint.py</code>, recall rebuild — were making hundreds of <em>serial</em>
requests, each opening a fresh TLS connection, and re-reading every note 2–3 times.
On the constrained agent sandbox that pushed a single pass past the ~120-second tool
timeout, so the process was killed mid-run and the session/thread was dropped. The
fix attacks the cause on three axes; the headline change is reliability:
<strong>a full-vault pass that used to fail now finishes in well under a second.</strong></p>
</section>
<section>
<h2>At a glance</h2>
<div class="metrics">
<div class="metric"><div class="num">4.5&times;</div><div class="lbl">faster per request<br>(connection reuse)</div></div>
<div class="metric"><div class="num">~2&times;</div><div class="lbl">added by concurrency<br>(full-vault scale)</div></div>
<div class="metric"><div class="num">2–3&times;</div><div class="lbl">fewer requests<br>(single-pass cache)</div></div>
<div class="metric"><div class="num good">&lt;1 s</div><div class="lbl">full-vault pass<br>(was: timeout)</div></div>
</div>
<p style="margin-top:6px">These factors compound. A full-vault pass pays
<strong>4.5&times;</strong> less per request, issues <strong>~2–3&times;</strong> fewer
of them, and runs the rest <strong>~2&times;</strong> in parallel — a modeled
<strong>~18–27&times;</strong> reduction in total request-time, before counting the
higher connection latency of the production sandbox where the old path actually failed.</p>
</section>
<section>
<h2>What changed</h2>
<div class="h2sub">Three structural fixes, one resolution overhaul</div>
<div class="grid g2">
<div class="card"><h4>Connection reuse (keep-alive)</h4><p>One persistent
connection per thread, reused across every request — instead of a fresh TCP+TLS
handshake per file. The single biggest win; benefits every operation
(load, capture, recall) for free.</p></div>
<div class="card"><h4>Concurrent bulk reads</h4><p>A new <code>read_many()</code>
fans GETs across a thread pool (8 workers). I/O-bound work, so it parallelizes
cleanly; the win grows with vault size and network latency.</p></div>
<div class="card"><h4>Single-pass shared cache</h4><p>Each note is fetched
<em>once</em> and reused across all passes — eliminating the 2–3&times; re-reads and
the per-link-target re-fetch storm in <code>sweep</code>.</p></div>
<div class="card"><h4>Entity resolution (v1.2.0)</h4><p>Shortened names now resolve
via aliases + a fuzzy fallback, so a mention like &ldquo;echo memory&rdquo; finds the
existing project instead of silently creating a duplicate note.</p></div>
</div>
</section>
<section>
<h2>Measured — per request</h2>
<div class="h2sub">Connection reuse is the dominant win</div>
<p>Micro-benchmark: 25 real notes, best of 3 runs, against the live endpoint.</p>
<div class="bars">
<div class="bar"><div class="name">Fresh connection / GET <em>(old)</em></div>
<div class="track"><div class="fill old" style="width:100%">30.3 ms</div></div></div>
<div class="bar"><div class="name">Pooled keep-alive (serial)</div>
<div class="track"><div class="fill" style="width:21.8%">6.6 ms</div></div></div>
<div class="bar"><div class="name">Pooled + concurrent <em>(new)</em></div>
<div class="track"><div class="fill best" style="width:18.8%">5.7 ms</div></div></div>
</div>
<table>
<tr><th>Mode</th><th class="n">ms / request</th><th class="n">vs old</th></tr>
<tr><td>Fresh connection per GET (old)</td><td class="n">30.3</td><td class="n">1.0&times;</td></tr>
<tr><td>Pooled keep-alive, serial</td><td class="n">6.6</td><td class="n delta">4.5&times;</td></tr>
<tr><td>Pooled + concurrent (new default)</td><td class="n">5.7</td><td class="n delta">5.3&times;</td></tr>
</table>
</section>
<section>
<h2>Measured — full vault (186 notes)</h2>
<div class="h2sub">Serial vs concurrent, both pooled</div>
<table>
<tr><th>Operation</th><th class="n">serial (1 worker)</th><th class="n">concurrent (8)</th><th class="n">speedup</th></tr>
<tr><td>vault_lint.py</td><td class="n">1.77 s</td><td class="n">0.90 s</td><td class="n delta">2.0&times;</td></tr>
<tr><td>sweep.py (plan)</td><td class="n">1.64 s</td><td class="n">0.85 s</td><td class="n delta">1.9&times;</td></tr>
<tr><td>sweep.py --apply</td><td class="n">&mdash;</td><td class="n">1.07 s</td><td class="n">&mdash;</td></tr>
<tr class="muted-row"><td>single GET · doctor</td><td class="n">&mdash;</td><td class="n">0.28 s · 0.21 s</td><td class="n">&mdash;</td></tr>
</table>
<div class="note warn"><strong>Before the fix:</strong> the same passes exceeded the
~120-second agent tool ceiling and were terminated mid-run, dropping the session.
The numbers above are the <em>new</em> floor; the practical change is
<strong>fails&nbsp;&rarr;&nbsp;completes</strong>.</div>
</section>
<section>
<h2>Efficiency — entity resolution (v1.2.0)</h2>
<div class="h2sub">Fewer duplicates, less wasted work</div>
<p>Resolution was exact-match only, so a shortened or expanded name returned
<em>nothing</em> — and the writer would create a parallel note. The active project
slugged <code>echo</code> was invisible to the mention &ldquo;echo memory.&rdquo; Now:</p>
<table>
<tr><th>Lookup</th><th>Before</th><th>After</th></tr>
<tr><td>echo memory · echo plugin · echo-memory</td><td><span class="delta" style="color:var(--bad)">no match</span></td><td class="delta">&rarr; projects/active/echo.md</td></tr>
<tr><td>non-aliased shortened name</td><td><span style="color:var(--bad);font-weight:700">no match</span></td><td class="delta">ranked candidates</td></tr>
<tr><td>update under a new name</td><td>second slug, same note</td><td class="delta">canonical slug + alias learned</td></tr>
</table>
<p style="margin-top:10px">Aliases are auto-derived from titles, learned from mentions on
update, and stored in note frontmatter (folded back into the index on sweep) — so the
graph gets <em>better</em> at resolution over time, while exact-match safety prevents
wrong auto-merges. The payoff is avoided duplicate entities and the cleanup/merge passes
they trigger.</p>
</section>
</div>
<div class="foot">
<p><b>Methodology.</b> Measured on the maintainer's workstation against the live
<code>echoapi.alwisp.com</code> endpoint (low latency). The production agent sandbox has
higher per-request connection latency, so the connection-reuse gain — and the reliability
fix — is larger there than the local figures show. The ~18–27&times; compound is a model of
the three measured factors (4.5&times; reuse &times; ~2&times; concurrency &times; ~2–3&times; fewer reads),
not a single end-to-end timing. Stdlib-only; no new dependencies.</p>
<p><b>Artifacts.</b> echo-memory v1.2.0 &mdash; <code>echo-memory-1.2.0.plugin</code>.
33 automated tests pass; vault-health linter clean of regressions.</p>
</div>
</div>
</body>
</html>
echo-memory-performance-brief.pdf
BIN
View File
Binary file not shown.
echo-memory.plugin
BIN
View File
Binary file not shown.
echo-memory.plugin.src/.claude-plugin/plugin.json
+2 -2
View File
@@ -1,7 +1,7 @@
{ {
"name": "echo-memory", "name": "echo-memory",
"version": "1.0.0", "version": "1.2.0",
"description": "Persistent memory via the ECHO Obsidian vault over the Obsidian Local REST API. Cross-platform Python client: one-call capture/resolve/recall/link over an entity index, hybrid BM25 + graph recall, auto-linking, an offline write-ahead queue + read cache, lock-guarded concurrency, and session-reflection capture, plus a linter-enforced routing manifest and /echo-load|save|recall|triage|health|sweep|reflect|doctor commands. Crosslinks notes across Claude/CoWork sessions.", "description": "Persistent memory via the ECHO Obsidian vault over the Obsidian Local REST API. Cross-platform Python client (connection-pooled, with concurrent full-vault reads): one-call capture/resolve/recall/link over an entity index, hybrid BM25 + graph recall, auto-linking, an offline write-ahead queue + read cache, lock-guarded concurrency, and session-reflection capture, plus a linter-enforced routing manifest and /echo-load|save|recall|triage|health|sweep|reflect|doctor commands. Crosslinks notes across Claude/CoWork sessions.",
"author": { "author": {
"name": "Jason" "name": "Jason"
}, },
echo-memory.plugin.src/skills/echo-memory/SKILL.md
+4 -2
View File
@@ -75,8 +75,8 @@ python3 "$ECHO" recall "<query>" # search + 1-hop link/sourc
python3 "$ECHO" link <pathA> <pathB> # add reciprocal [[Related]] links A<->B python3 "$ECHO" link <pathA> <pathB> # add reciprocal [[Related]] links A<->B
``` ```
- **`capture` is the default write.** Given a title + `--kind`, it: resolves the entity via the index (create-vs-update, no manual search-first), derives the canonical path, stamps canonical frontmatter (`type/created/updated/agent_written/source_notes`), updates the index, **auto-links** any known entity mentioned in the body (bidirectionally), and appends today's Agent Log line — one call instead of search→put→bump→log. `--kind` ∈ `person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`. Use `--inbox` (or omit `--kind`) only when the home is genuinely unknown. - **`capture` is the default write.** Given a title + `--kind`, it: resolves the entity via the index (create-vs-update, no manual search-first), derives the canonical path, stamps canonical frontmatter (`type/created/updated/aliases/agent_written/source_notes`), updates the index, **auto-links** any known entity mentioned in the body (bidirectionally), and appends today's Agent Log line — one call instead of search→put→bump→log. It also **auto-derives aliases** from the title, **learns the mention as an alias** when updating an entity under a different name, and **warns if the new note resembles an existing entity** (so a shortened name can't silently spawn a duplicate). `--kind` ∈ `person, company, concept, reference, meeting, project, area, semantic, episodic, working, skill, decision`. Use `--inbox` (or omit `--kind`) only when the home is genuinely unknown.
- **`resolve` before constructing any path by hand** — it returns the canonical note (matching aliases, so *Bob/Robert/RS* converge) or tells you where a new one goes. This replaces the two-search "search-first" dance for slug-addressed notes. - **`resolve` before constructing any path by hand** — it returns the canonical note (matching slug/title/alias, so *Bob/Robert/RS* converge) or, when nothing matches exactly, a ranked list of **`candidates`** (entities sharing a distinctive token — e.g. "echo memory" surfaces the project `echo`). Always check candidates before creating a new note; a shortened name not matching exactly is the classic way a duplicate gets made. This replaces the two-search "search-first" dance for slug-addressed notes.
- **`recall` for "what do we know about X"** — it returns the matching notes *and* their one-hop neighbourhood (Related links, `source_notes`), so recall surfaces the web around a topic, not an isolated note. - **`recall` for "what do we know about X"** — it returns the matching notes *and* their one-hop neighbourhood (Related links, `source_notes`), so recall surfaces the web around a topic, not an isolated note.
- **`link`** when two existing notes are related but not yet connected; `capture` already auto-links what it detects. - **`link`** when two existing notes are related but not yet connected; `capture` already auto-links what it detects.
@@ -311,6 +311,8 @@ Run it after `migrate.py` (which handles structural/schema changes) — or any t
The pass is cheap and pays for itself by catching drift before it requires a reorg. Write the findings as a digest; act on them only with Jason's go-ahead. The pass is cheap and pays for itself by catching drift before it requires a reorg. Write the findings as a digest; act on them only with Jason's go-ahead.
**Performance.** Full-vault scripts (`sweep.py`, `vault_lint.py`, recall rebuild) are connection-pooled (keep-alive) and read the whole vault concurrently via `echo.read_many`, so they finish in about a second on a few-hundred-note vault instead of timing out on hundreds of serial TLS handshakes. Tune with `ECHO_WORKERS` (default 8, bulk-read concurrency) and `ECHO_TIMEOUT` (default 30s, per-request). For a vault large enough that even the concurrent pass nears the tool timeout, run the script with the harness's background-execution option rather than blocking the turn.
## Daily Note — Agent Log ## Daily Note — Agent Log
After substantive activity, write a one-line entry to today's daily note's `## Agent Log` heading. The daily-note **template** (`journal/templates/daily-note-template.md`) defines this heading, but ad-hoc daily notes created without the template don't have it — so PATCH can fail with `invalid-target` if the note exists but lacks the heading. After substantive activity, write a one-line entry to today's daily note's `## Agent Log` heading. The daily-note **template** (`journal/templates/daily-note-template.md`) defines this heading, but ad-hoc daily notes created without the template don't have it — so PATCH can fail with `invalid-target` if the note exists but lacks the heading.
echo-memory.plugin.src/skills/echo-memory/references/api-reference.md
+15
View File
@@ -8,6 +8,21 @@ The endpoint has a **valid TLS certificate** — `-k` is not required. Paths add
--- ---
## Client performance (echo.py) — pooling & concurrency
`echo.request()` keeps **one persistent keep-alive connection per thread** and reuses it across calls, instead of opening a fresh TCP+TLS connection per request. Against a remote HTTPS endpoint the repeated handshake was the dominant cost, so a full-vault pass used to make hundreds of serial handshakes and exceed the agent/tool timeout (which kills the run mid-flight). A stale pooled connection reconnects transparently; the `(status, body)` contract is unchanged (status `0` still means transport failure, so the offline write-queue still triggers).
For bulk reads, **`echo.read_many(paths)`** fans GETs across a thread pool and returns `{path: text_or_None}` — resilient (one unreadable file becomes `None` rather than aborting the batch). `sweep.py` and `vault_lint.py` prefetch the whole vault once with it and share that single cache across every pass, so no note is fetched more than once. Net effect: a several-minute full-vault sweep becomes sub-second on a few-hundred-note vault.
Tuning knobs (env overrides):
- `ECHO_TIMEOUT` (default `30`) — per-request socket timeout in seconds. With concurrency, one slow file only blocks its own worker, not the run.
- `ECHO_WORKERS` (default `8`) — `read_many` thread-pool size. Raise it for very large vaults; lower it to be gentler on the backend.
For a vault large enough that even the concurrent pass approaches the tool timeout, run the script with the harness's background-execution option rather than blocking the turn.
---
## Reading Files ## Reading Files
```bash ```bash
echo-memory.plugin.src/skills/echo-memory/references/vault-layout.md
+5 -1
View File
@@ -51,7 +51,7 @@
└── locks/ ← vault.lock — cooperative advisory multi-writer lock └── locks/ ← vault.lock — cooperative advisory multi-writer lock
``` ```
**Entity index:** `_agent/index/entities.json` is a machine-maintained registry mapping each entity slug to its `{path, kind, title, aliases, last_seen}`. It makes routing/resolve an O(1), alias-aware lookup (no fuzzy search per write) and supplies the name→path map for cross-linking and recall. It is rebuilt automatically by `echo.py capture` and by `sweep.py`; do not hand-edit. The only JSON file in the vault. **Entity index:** `_agent/index/entities.json` is a machine-maintained registry mapping each entity slug to its `{path, kind, title, aliases, last_seen}`. It makes routing/resolve an O(1), alias-aware lookup and supplies the name→path map for cross-linking and recall. `resolve` matches slug/title/alias exactly; when nothing matches exactly it falls back to **fuzzy candidates** — entities sharing a distinctive token with the mention — so a shortened name ("echo memory" → the project `echo`) surfaces the existing note instead of spawning a duplicate. Aliases are auto-derived from titles, learned from mentions on update, and re-folded from note frontmatter on `sweep`. It is rebuilt automatically by `echo.py capture` and by `sweep.py`; do not hand-edit. The only JSON file in the vault.
**Heartbeat:** `_agent/heartbeat/last-session.md` is a one-line pointer (`<session-log-path> @ <ISO-timestamp>`) the **session-logging procedure writes (PUT, overwrite) at session end** and the **loading procedure reads first (Step 4)** as an O(1) shortcut to the latest session log. It is a hint, not a source of truth — fall back to the `sessions/` directory listing if it's missing or stale. Because it's PUT-overwritten, it never grows or duplicates. **Heartbeat:** `_agent/heartbeat/last-session.md` is a one-line pointer (`<session-log-path> @ <ISO-timestamp>`) the **session-logging procedure writes (PUT, overwrite) at session end** and the **loading procedure reads first (Step 4)** as an O(1) shortcut to the latest session log. It is a hint, not a source of truth — fall back to the `sessions/` directory listing if it's missing or stale. Because it's PUT-overwritten, it never grows or duplicates.
@@ -70,11 +70,15 @@ status: # active | draft | done | archived | complete
created: # YYYY-MM-DD (or YYYY-MM-DDTHH:mm for sessions) created: # YYYY-MM-DD (or YYYY-MM-DDTHH:mm for sessions)
updated: # YYYY-MM-DD updated: # YYYY-MM-DD
tags: [] tags: []
aliases: [] # optional — other names this entity is called (Obsidian-native). Folded
# into the entity index so a shortened/expanded name resolves to this note.
agent_written: false agent_written: false
source_notes: [] # plain relative paths as strings — NEVER [[wikilinks]] source_notes: [] # plain relative paths as strings — NEVER [[wikilinks]]
--- ---
``` ```
`aliases:` is the **durable, Obsidian-native home for alternate names** of an entity (e.g. the project `echo` is also "echo-memory" / "echo plugin"). `capture` auto-derives the obvious case/punctuation variants of the title and writes them here; `sweep` folds frontmatter aliases back into the entity index, so they survive an index rebuild. List plain strings, never `[[wikilinks]]`.
`agent_written: true` + a populated `source_notes` is the key signal separating `agent_written: true` + a populated `source_notes` is the key signal separating
agent-managed content from human-authored content. When appending with POST, do agent-managed content from human-authored content. When appending with POST, do
not rewrite frontmatter — the append goes after existing content. To change not rewrite frontmatter — the append goes after existing content. To change
echo-memory.plugin.src/skills/echo-memory/scripts/echo.py
+101 -13
View File
@@ -14,11 +14,18 @@ existing one is no longer wrongly skipped), correct `::` heading-target handling
frontmatter field patches, a read-back-confirmed advisory multi-writer lock, a frontmatter field patches, a read-back-confirmed advisory multi-writer lock, a
one-call cold-start `load`, and scope show/set. one-call cold-start `load`, and scope show/set.
Network is keep-alive and connection-pooled (one persistent connection per thread,
reused across requests), and `read_many` fans bulk GETs across a thread pool — so
full-vault scripts (sweep, lint, recall rebuild) make a handful of warm round-trips
instead of hundreds of fresh TLS handshakes, and no longer blow past tool timeouts.
Config (env overrides; defaults match the rest of the plugin): Config (env overrides; defaults match the rest of the plugin):
ECHO_BASE default https://echoapi.alwisp.com ECHO_BASE default https://echoapi.alwisp.com
ECHO_KEY default the plugin bearer token (env overrides it) ECHO_KEY default the plugin bearer token (env overrides it)
ECHO_VERIFY default 1 — read-back verify after a PUT ECHO_VERIFY default 1 — read-back verify after a PUT
ECHO_LOCK_TTL default 900 — seconds before an advisory lock is considered stale ECHO_LOCK_TTL default 900 — seconds before an advisory lock is considered stale
ECHO_TIMEOUT default 30 — per-request socket timeout (seconds)
ECHO_WORKERS default 8 — concurrency for read_many bulk reads
ECHO_TODAY YYYY-MM-DD to use as "today" (pass the conversation's currentDate) ECHO_TODAY YYYY-MM-DD to use as "today" (pass the conversation's currentDate)
Usage: Usage:
@@ -46,15 +53,16 @@ from __future__ import annotations
import argparse import argparse
import datetime as dt import datetime as dt
import http.client
import json import json
import locale import locale
import os import os
import sys import sys
import tempfile import tempfile
import threading
import time import time
import urllib.error
import urllib.parse import urllib.parse
import urllib.request from concurrent.futures import ThreadPoolExecutor
from pathlib import Path from pathlib import Path
# Never let a non-ASCII byte (em-dash in our own output, or Unicode in a fetched # Never let a non-ASCII byte (em-dash in our own output, or Unicode in a fetched
@@ -76,6 +84,11 @@ DEFAULT_KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
KEY = echo_secrets.resolve_key(DEFAULT_KEY) KEY = echo_secrets.resolve_key(DEFAULT_KEY)
VERIFY = os.environ.get("ECHO_VERIFY", "1") != "0" VERIFY = os.environ.get("ECHO_VERIFY", "1") != "0"
LOCK_TTL = int(os.environ.get("ECHO_LOCK_TTL", "900")) LOCK_TTL = int(os.environ.get("ECHO_LOCK_TTL", "900"))
# Per-request socket timeout. A single stuck file fails fast instead of stalling a
# full-vault pass; with read_many concurrency it only blocks one worker, not the run.
TIMEOUT = int(os.environ.get("ECHO_TIMEOUT", "30"))
# Concurrency for bulk reads (read_many). I/O-bound, so threads bypass the GIL fine.
MAX_WORKERS = max(1, int(os.environ.get("ECHO_WORKERS", "8")))
LOCK_PATH = "_agent/locks/vault.lock" LOCK_PATH = "_agent/locks/vault.lock"
@@ -101,30 +114,105 @@ def vault_url(path: str) -> str:
return f"{BASE}/vault/{safe}" return f"{BASE}/vault/{safe}"
# --- persistent connection pool (keep-alive) ---------------------------------
# urllib.urlopen opened a fresh TCP+TLS connection per call; against a remote HTTPS
# endpoint the repeated handshake dominated, so a full-vault pass made hundreds of
# serial handshakes and blew past the tool/sandbox timeout. We keep ONE connection
# per thread (http.client is not thread-safe, so each thread — main + every pool
# worker — owns its own) and reuse it across requests. read_many fans out over a
# thread pool, so the pool's connections stay warm for the duration of a sweep.
_local = threading.local()
MAX_ATTEMPTS = 3
def _new_connection() -> http.client.HTTPConnection:
parts = urllib.parse.urlsplit(BASE)
host = parts.hostname or parts.path
if parts.scheme == "http":
return http.client.HTTPConnection(host, parts.port or 80, timeout=TIMEOUT)
return http.client.HTTPSConnection(host, parts.port or 443, timeout=TIMEOUT)
def _connection() -> http.client.HTTPConnection:
conn = getattr(_local, "conn", None)
if conn is None:
conn = _new_connection()
_local.conn = conn
return conn
def _drop_connection() -> None:
conn = getattr(_local, "conn", None)
if conn is not None:
try:
conn.close()
except Exception:
pass
_local.conn = None
def request(method: str, url: str, data: bytes | None = None, def request(method: str, url: str, data: bytes | None = None,
headers: dict[str, str] | None = None) -> tuple[int, bytes]: headers: dict[str, str] | None = None) -> tuple[int, bytes]:
"""One bounded retry on transport failure (status 0) or 5xx. Returns (status, body).""" """Issue one request over this thread's reused connection. Returns (status, body);
status 0 == transport failure (echo_queue relies on this to trigger offline queueing).
Retries: a stale pooled connection reconnects immediately; 5xx and other transport
errors get one bounded sleep-retry, same as before."""
merged = {"Authorization": f"Bearer {KEY}", **(headers or {})} merged = {"Authorization": f"Bearer {KEY}", **(headers or {})}
parts = urllib.parse.urlsplit(url)
target = parts.path or "/"
if parts.query:
target += "?" + parts.query
last_status, last_body = 0, b"" last_status, last_body = 0, b""
for attempt in range(2): for attempt in range(MAX_ATTEMPTS):
req = urllib.request.Request(url, data=data, method=method, headers=merged)
try: try:
with urllib.request.urlopen(req, timeout=30) as response: conn = _connection()
return response.status, response.read() conn.request(method, target, body=data, headers=merged)
except urllib.error.HTTPError as exc: resp = conn.getresponse()
body = exc.read() body = resp.read() # drain fully so the connection can be reused
if exc.code >= 500 and attempt == 0: status = resp.status
if status >= 500 and attempt < MAX_ATTEMPTS - 1:
_drop_connection()
time.sleep(1) time.sleep(1)
continue continue
return exc.code, body return status, body
except Exception as exc: # connection error, timeout, DNS, ... except (http.client.RemoteDisconnected, http.client.BadStatusLine,
ConnectionResetError, BrokenPipeError) as exc:
# The server closed an idle keep-alive connection between requests.
# Expected, not a failure: reconnect and retry without backing off.
_drop_connection()
last_status, last_body = 0, str(exc).encode() last_status, last_body = 0, str(exc).encode()
if attempt == 0: continue
except Exception as exc: # timeout, DNS, refused, TLS, ...
_drop_connection()
last_status, last_body = 0, str(exc).encode()
if attempt < MAX_ATTEMPTS - 1:
time.sleep(1) time.sleep(1)
continue continue
return last_status, last_body return last_status, last_body
def get_text(path: str) -> str | None:
"""GET a vault path -> decoded text, or None on 404/any error. Never raises, so a
single unreadable note can't abort a bulk pass (resilience for sweep/lint)."""
try:
status, body = request("GET", vault_url(path))
except Exception:
return None
return body.decode(errors="replace") if status == 200 else None
def read_many(paths) -> dict[str, str | None]:
"""Concurrently GET many vault paths -> {path: text_or_None}. The single biggest
win for full-vault scripts: it turns hundreds of serial round-trips into MAX_WORKERS
parallel ones over warm keep-alive connections. Resilient — a bad file maps to None."""
unique = list(dict.fromkeys(paths))
if not unique:
return {}
workers = min(MAX_WORKERS, len(unique))
with ThreadPoolExecutor(max_workers=workers) as pool:
return dict(zip(unique, pool.map(get_text, unique)))
def check(status: int, body: bytes, context: str) -> None: def check(status: int, body: bytes, context: str) -> None:
if status == 404: if status == 404:
raise EchoError(f"{context}: not found", 44) raise EchoError(f"{context}: not found", 44)
echo-memory.plugin.src/skills/echo-memory/scripts/echo_index.py
+74 -2
View File
@@ -26,6 +26,7 @@ from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.path.insert(0, str(Path(__file__).resolve().parent))
import echo # noqa: E402 import echo # noqa: E402
import echo_quality # noqa: E402 — distinctive-token guards for fuzzy candidate matching
INDEX_PATH = "_agent/index/entities.json" INDEX_PATH = "_agent/index/entities.json"
@@ -61,6 +62,48 @@ def slugify(text: str) -> str:
return text[:40] or "untitled" return text[:40] or "untitled"
def derive_aliases(title: str) -> list[str]:
"""Safe, lossless name variants of a title — case/punctuation normalizations that are
unambiguously the SAME name (hyphen<->space, lowercased). Deliberately NOT acronyms or
token subsets: those collide across entities and would make resolve() match too much.
Lets `echo-memory` and `echo memory` both resolve to a title written either way."""
t = str(title or "").strip()
if not t:
return []
low = t.lower()
variants = {low, slugify(t), slugify(t).replace("-", " "),
low.replace("-", " "), low.replace(" ", "-")}
return sorted(v for v in variants if v)
_WORD = re.compile(r"[a-z0-9]+")
def _sig_tokens(s) -> set[str]:
"""Distinctive tokens of a string: long enough and not a common word (reuses the
auto-link guards), so 'data'/'api'/'the' can't drive a fuzzy match on their own."""
return {t for t in _WORD.findall(str(s or "").lower())
if len(t) >= echo_quality.MIN_NAME_LEN and t not in echo_quality._COMMON}
def aliases_in_frontmatter(text: str) -> list[str]:
"""Read an `aliases:` list (flow `[a, b]` or block `- a`) from a note's YAML frontmatter.
Frontmatter is the durable, Obsidian-native home for aliases; sweep folds these into the
index so they survive an index rebuild."""
if not str(text).startswith("---"):
return []
end = text.find("\n---", 3)
fm = text[:end] if end != -1 else text
flow = re.search(r"(?m)^aliases:\s*\[(.*?)\]", fm)
if flow:
return [a.strip().strip('"').strip("'") for a in flow.group(1).split(",") if a.strip()]
block = re.search(r"(?m)^aliases:\s*\n((?:[ \t]*-[ \t]*.+\n?)+)", fm)
if block:
return [ln.strip().lstrip("-").strip().strip('"').strip("'")
for ln in block.group(1).splitlines() if ln.strip()]
return []
def derive_path(kind: str, slug: str, date: str | None = None, domain: str = "business") -> str: def derive_path(kind: str, slug: str, date: str | None = None, domain: str = "business") -> str:
folder = KIND_FOLDER.get(kind) folder = KIND_FOLDER.get(kind)
if folder is None: if folder is None:
@@ -128,7 +171,9 @@ def _norm(s) -> str:
def resolve(index: dict, mention: str): def resolve(index: dict, mention: str):
"""Return (slug, entry) for a mention matched by slug, title, or alias — else (None, None).""" """Return (slug, entry) for a mention EXACTLY matched by slug, title, or alias — else
(None, None). Exact-only by design: this drives capture's create-vs-update decision, so a
loose match here would silently merge distinct entities. Fuzzy lookup is fuzzy_candidates()."""
key = _norm(mention) key = _norm(mention)
ents = index.get("entities", {}) ents = index.get("entities", {})
if key in ents: if key in ents:
@@ -143,6 +188,29 @@ def resolve(index: dict, mention: str):
return None, None return None, None
def fuzzy_candidates(index: dict, mention: str, limit: int = 5):
"""Advisory near-matches for when resolve() finds no exact hit. Ranks entities sharing
>=1 *distinctive* token (>= MIN_NAME_LEN, not a common word) with the mention, so a
shortened or expanded name — 'echo memory' for the project 'echo' — surfaces the existing
note instead of vanishing. Returns [(slug, entry, score)] sorted best-first. NEVER used to
auto-merge (that's resolve's job); only to SURFACE 'did you mean' candidates."""
mtoks = _sig_tokens(mention)
if not mtoks:
return []
scored = []
for slug, e in index.get("entities", {}).items():
etoks = set()
for name in (slug, e.get("title", ""), *e.get("aliases", [])):
etoks |= _sig_tokens(name)
overlap = mtoks & etoks
if not overlap or not etoks:
continue
score = round(len(overlap) / min(len(mtoks), len(etoks)), 3)
scored.append((score, len(overlap), slug, e))
scored.sort(key=lambda x: (-x[0], -x[1], x[2]))
return [(slug, e, score) for score, _, slug, e in scored[:limit]]
def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = None, def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = None,
aliases=None) -> dict: aliases=None) -> dict:
ents = index.setdefault("entities", {}) ents = index.setdefault("entities", {})
@@ -150,7 +218,11 @@ def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = Non
e["path"] = path e["path"] = path
e["kind"] = kind e["kind"] = kind
e["title"] = title or e.get("title") or slug e["title"] = title or e.get("title") or slug
merged = set(e.get("aliases", [])) | set(aliases or []) # Keep any prior + explicit aliases, and ALWAYS fold in the safe title variants so a
# hyphen/space/case form of the name resolves even if no one passed --aliases. Drop any
# alias that just equals the slug (resolve already checks the slug).
merged = set(e.get("aliases", [])) | set(aliases or []) | set(derive_aliases(e["title"]))
merged.discard(slug)
e["aliases"] = sorted(a for a in merged if a) e["aliases"] = sorted(a for a in merged if a)
e["last_seen"] = echo.today() e["last_seen"] = echo.today()
ents[slug] = e ents[slug] = e
echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py
+52 -15
View File
@@ -36,11 +36,19 @@ def resolve(mention: str) -> int:
slug, e = idx_mod.resolve(index, mention) slug, e = idx_mod.resolve(index, mention)
if e: if e:
print(json.dumps({"match": True, "slug": slug, **e}, ensure_ascii=False, indent=2)) print(json.dumps({"match": True, "slug": slug, **e}, ensure_ascii=False, indent=2))
return 0
# No exact match — surface fuzzy candidates so a shortened/expanded name (e.g. "echo
# memory" for the project "echo") reveals the existing note instead of looking absent.
cands = idx_mod.fuzzy_candidates(index, mention)
out = {"match": False, "mention": mention, "suggest_slug": idx_mod.slugify(mention)}
if cands:
out["candidates"] = [{"slug": s, "path": c.get("path"), "title": c.get("title"),
"kind": c.get("kind"), "score": sc} for s, c, sc in cands]
out["note"] = ("no exact match, but similar entities exist (see candidates) — check "
"these before creating a new note; reuse the right path or `echo.py link`.")
else: else:
print(json.dumps({"match": False, "mention": mention, out["note"] = "no index entry — derive a path with the right --kind and create it"
"suggest_slug": idx_mod.slugify(mention), print(json.dumps(out, ensure_ascii=False, indent=2))
"note": "no index entry — derive a path with the right --kind and create it"},
ensure_ascii=False, indent=2))
return 0 return 0
@@ -93,13 +101,16 @@ def ensure_daily_log(line: str) -> None:
# ----------------------------------------------------------------- capture --- # ----------------------------------------------------------------- capture ---
def _build_note(fm_type: str, status_v: str, today_s: str, title: str, def _build_note(fm_type: str, status_v: str, today_s: str, title: str,
body_text: str, sources: list[str]) -> str: body_text: str, sources: list[str], aliases: list[str] | None = None) -> str:
src = "[" + ", ".join(json.dumps(s) for s in sources) + "]" src = "[" + ", ".join(json.dumps(s) for s in sources) + "]"
fm = ["---", f"type: {fm_type}"] fm = ["---", f"type: {fm_type}"]
if status_v: if status_v:
fm.append(f"status: {status_v}") fm.append(f"status: {status_v}")
fm += [f"created: {today_s}", f"updated: {today_s}", "tags: []", fm += [f"created: {today_s}", f"updated: {today_s}", "tags: []"]
"agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""] if aliases:
# Obsidian-native, durable home for aliases; sweep folds these back into the index.
fm.append("aliases: [" + ", ".join(json.dumps(a) for a in aliases) + "]")
fm += ["agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""]
out = "\n".join(fm) out = "\n".join(fm)
if body_text.strip(): if body_text.strip():
out += body_text.strip() + "\n" out += body_text.strip() + "\n"
@@ -141,11 +152,14 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
# M4: in --json mode, swallow the helper "ok:" chatter so stdout is clean JSON. # M4: in --json mode, swallow the helper "ok:" chatter so stdout is clean JSON.
return contextlib.redirect_stdout(io.StringIO()) if as_json else contextlib.nullcontext() return contextlib.redirect_stdout(io.StringIO()) if as_json else contextlib.nullcontext()
def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False) -> int: def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False,
near=None) -> int:
if as_json: if as_json:
act = f"dry-run:{action}" if dry else action act = f"dry-run:{action}" if dry else action
env = echo_output.envelope(act, {"kind": kind, "path": path, "title": title, data = {"kind": kind, "path": path, "title": title, "links_added": links}
"links_added": links}, ok=ok) if near:
data["near_duplicates"] = near
env = echo_output.envelope(act, data, ok=ok)
print(json.dumps(env, ensure_ascii=False), file=real_stdout) print(json.dumps(env, ensure_ascii=False), file=real_stdout)
elif dry: elif dry:
print(f"would {action} {kind or '-'} -> {path}") print(f"would {action} {kind or '-'} -> {path}")
@@ -170,29 +184,47 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
slug = idx_mod.slugify(title) slug = idx_mod.slugify(title)
index = idx_mod.load() index = idx_mod.load()
_, existing = idx_mod.resolve(index, title) match_slug, existing = idx_mod.resolve(index, title)
existing_reachable = bool(existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200) existing_reachable = bool(existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200)
if dry_run: if dry_run:
if existing_reachable: if existing_reachable:
return done("update", existing["path"], dry=True) return done("update", existing["path"], dry=True)
s2 = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug) s2 = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
return done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain), dry=True) near = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
return done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain),
dry=True, near=near)
near_dupes: list[str] = []
index_title = title # title to record in the index entry
index_aliases = list(aliases)
with quiet(): with quiet():
if existing_reachable: if existing_reachable:
# Reuse the matched entity's CANONICAL slug — never re-slug from the mention, or
# two index slugs end up pointing at one note. Keep its title; learn the mention
# as an alias when it's a new distinctive form, so this name resolves next time.
slug = match_slug or slug
path = existing["path"] path = existing["path"]
kind = existing.get("kind", kind) kind = existing.get("kind", kind)
index_title = existing.get("title") or title
known = {idx_mod._norm(a) for a in existing.get("aliases", [])} | {match_slug}
if idx_mod.slugify(title) not in known and len(title.strip()) >= echo_quality.MIN_NAME_LEN:
index_aliases.append(title)
_append_to_existing(path, kind, today_s, body_text) _append_to_existing(path, kind, today_s, body_text)
action = "updated" action = "updated"
else: else:
if not status_v and kind == "project": if not status_v and kind == "project":
status_v = "active" status_v = "active"
# Surface (don't silently create alongside) an existing entity this resembles —
# the duplication trap when a shortened name didn't resolve exactly.
near_dupes = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
# M2: don't let a 40-char-truncated slug silently collide with a different # M2: don't let a 40-char-truncated slug silently collide with a different
# entity already in the index — disambiguate to slug-2, slug-3, ... # entity already in the index — disambiguate to slug-2, slug-3, ...
slug = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug) slug = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
path = idx_mod.derive_path(kind, slug, date=date, domain=domain) path = idx_mod.derive_path(kind, slug, date=date, domain=domain)
note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title, body_text, sources) note_aliases = sorted((set(index_aliases) | set(idx_mod.derive_aliases(title))) - {slug})
note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title,
body_text, sources, note_aliases)
echo.cmd_put(path, echo.temp_file(note.encode())) echo.cmd_put(path, echo.temp_file(note.encode()))
action = "created" action = "created"
@@ -201,7 +233,7 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
# fresh-re-read transaction. Returns the fresh index (used by auto-link below). # fresh-re-read transaction. Returns the fresh index (used by auto-link below).
import echo_concurrency import echo_concurrency
index = echo_concurrency.atomic_index_update( index = echo_concurrency.atomic_index_update(
lambda idx: idx_mod.upsert(idx, slug, path, kind, title, aliases)) lambda idx: idx_mod.upsert(idx, slug, path, kind, index_title, index_aliases))
# auto-link: any other known entity CONFIDENTLY named in the body (M2: is_confident_link # auto-link: any other known entity CONFIDENTLY named in the body (M2: is_confident_link
# rejects short/common tokens so a 3-char alias or a word like "API" can't link everywhere). # rejects short/common tokens so a 3-char alias or a word like "API" can't link everywhere).
@@ -226,7 +258,12 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]") ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]")
if as_json: if as_json:
done(action, path, links=linked) done(action, path, links=linked, near=near_dupes)
else: else:
print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else "")) print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else ""))
if near_dupes:
kind_word = "entity" if len(near_dupes) == 1 else "entities"
print(f"WARNING: similar existing {kind_word} ({', '.join(near_dupes)}) — if this is "
f"the same thing, merge or `echo.py link` instead of keeping a duplicate.",
file=real_stdout)
return 0 return 0
echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py
+10 -6
View File
@@ -195,14 +195,18 @@ def save_index(ix: Bm25Index) -> None:
echo.check(status, b, "recall-index save") echo.check(status, b, "recall-index save")
def rebuild() -> Bm25Index: def rebuild(prefetched: dict | None = None) -> Bm25Index:
"""Full rebuild from every indexable entity note. Called by sweep.py and as the """Full rebuild from every indexable entity note. Called by sweep.py and as the
cold-cache path inside recall(). Saves and returns the index.""" cold-cache path inside recall(). Saves and returns the index.
`prefetched` (a {path: text} map, e.g. from echo.read_many) lets sweep.py reuse the
content it already pulled instead of walking and re-fetching the whole vault again."""
ix = Bm25Index() ix = Bm25Index()
for path in _walk(): if prefetched is not None:
if not _indexable(path): items = ((p, t) for p, t in prefetched.items() if _indexable(p))
continue else:
text = _get(path) items = ((p, _get(p)) for p in _walk() if _indexable(p))
for path, text in items:
if text is not None: if text is not None:
ix.add(path, strip_frontmatter(text)) ix.add(path, strip_frontmatter(text))
save_index(ix) save_index(ix)
echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py
+14 -5
View File
@@ -99,6 +99,12 @@ def main(argv: list[str] | None = None) -> int:
if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
and not TEMPLATE_RE.search(p)] and not TEMPLATE_RE.search(p)]
# Fetch every note's content ONCE, concurrently. All three passes below (index,
# recall, link symmetrize) read from this shared cache, so no file is fetched
# twice and link targets aren't re-fetched per reference.
texts = echo.read_many(all_files)
print(f"sweep: read {len(texts)} notes (concurrent, keep-alive)\n")
# ---- 2. (re)build entity index ------------------------------------------- # ---- 2. (re)build entity index -------------------------------------------
index = idx_mod.load() index = idx_mod.load()
existing = dict(index.get("entities", {})) existing = dict(index.get("entities", {}))
@@ -110,10 +116,13 @@ def main(argv: list[str] | None = None) -> int:
continue continue
base = path.rsplit("/", 1)[-1][:-3] base = path.rsplit("/", 1)[-1][:-3]
slug = idx_mod.slugify(base) slug = idx_mod.slugify(base)
text = get(path) or "" text = texts.get(path) or ""
title = links.first_h1(text) or base title = links.first_h1(text) or base
prev = existing.get(slug) prev = existing.get(slug)
aliases = prev.get("aliases", []) if prev else [] # Frontmatter aliases are authoritative and re-folded every rebuild; prior index
# aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
prev_aliases = prev.get("aliases", []) if prev else []
aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
idx_mod.upsert(rebuilt, slug, path, kind, title, aliases) idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
if not prev: if not prev:
added += 1 added += 1
@@ -126,7 +135,7 @@ def main(argv: list[str] | None = None) -> int:
# ---- 2b. (re)build the recall (BM25) index ------------------------------- # ---- 2b. (re)build the recall (BM25) index -------------------------------
if apply: if apply:
rix = echo_recall.rebuild() rix = echo_recall.rebuild(prefetched=texts)
print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)") print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)")
else: else:
n_idx = sum(1 for p in all_files if echo_recall._indexable(p)) n_idx = sum(1 for p in all_files if echo_recall._indexable(p))
@@ -147,14 +156,14 @@ def main(argv: list[str] | None = None) -> int:
missing = set() # (target_path, source_path) — reciprocal link to add on target missing = set() # (target_path, source_path) — reciprocal link to add on target
for path in all_files: for path in all_files:
text = get(path) text = texts.get(path)
if text is None: if text is None:
continue continue
for tgt in links.related_targets(text): for tgt in links.related_targets(text):
tp = resolve_target(tgt) tp = resolve_target(tgt)
if not tp or tp == path: if not tp or tp == path:
continue continue
back = {resolve_target(t) for t in links.related_targets(get(tp) or "")} back = {resolve_target(t) for t in links.related_targets(texts.get(tp) or "")}
if path not in back: if path not in back:
missing.add((tp, path)) missing.add((tp, path))
missing = sorted(missing) missing = sorted(missing)
echo-memory.plugin.src/skills/echo-memory/scripts/test_echo_client.py
+61
View File
@@ -38,6 +38,29 @@ def test_frontmatter_existing_json_is_preserved() -> None:
assert echo.normalize_json_scalar('"2026-06-20"') == b'"2026-06-20"' assert echo.normalize_json_scalar('"2026-06-20"') == b'"2026-06-20"'
def test_read_many_dedups_and_maps_each_path() -> None:
# Concurrency helper contract, no network: monkeypatch the per-file fetch.
orig = echo.get_text
echo.get_text = lambda p: f"T:{p}"
try:
assert echo.read_many(["a", "b", "a"]) == {"a": "T:a", "b": "T:b"}
finally:
echo.get_text = orig
def test_read_many_empty_returns_empty_dict() -> None:
assert echo.read_many([]) == {}
def test_read_many_tolerates_unreadable_file_as_none() -> None:
orig = echo.get_text
echo.get_text = lambda p: None if p == "bad" else f"T:{p}"
try:
assert echo.read_many(["ok", "bad"]) == {"ok": "T:ok", "bad": None}
finally:
echo.get_text = orig
def test_stem_extracts_literal_directory_prefix() -> None: def test_stem_extracts_literal_directory_prefix() -> None:
assert check_routing.stem(r"^projects/active/[^/]+\.md$") == "projects/active/" assert check_routing.stem(r"^projects/active/[^/]+\.md$") == "projects/active/"
assert check_routing.stem(r"^areas/(business|personal)/[^/]+\.md$") == "areas/" assert check_routing.stem(r"^areas/(business|personal)/[^/]+\.md$") == "areas/"
@@ -86,6 +109,44 @@ def test_kind_for_path() -> None:
assert echo_index.kind_for_path("journal/daily/2026-01-01.md") is None assert echo_index.kind_for_path("journal/daily/2026-01-01.md") is None
def test_derive_aliases_produces_punctuation_variants() -> None:
al = echo_index.derive_aliases("ECHO Memory")
assert "echo-memory" in al and "echo memory" in al
assert echo_index.derive_aliases("") == []
def test_upsert_folds_in_title_variants_and_drops_slug() -> None:
index = {"entities": {}}
e = echo_index.upsert(index, "echo-memory", "projects/active/echo-memory.md",
"project", "Echo Memory")
assert "echo memory" in e["aliases"] # space variant auto-derived
assert "echo-memory" not in e["aliases"] # equals the slug -> not stored redundantly
def test_fuzzy_candidates_finds_shortened_name() -> None:
# The real bug: a project slugged "echo" must surface for the mention "echo memory".
index = {"entities": {"echo": {"path": "projects/active/echo.md", "kind": "project",
"title": "echo", "aliases": []}}}
cands = echo_index.fuzzy_candidates(index, "echo memory")
assert cands and cands[0][0] == "echo"
# exact resolve still misses (it's exact-only) — fuzzy is the safety net:
assert echo_index.resolve(index, "echo memory")[0] is None
def test_fuzzy_candidates_ignores_common_and_short_tokens() -> None:
index = {"entities": {"data": {"path": "x.md", "kind": "concept",
"title": "data", "aliases": []}}}
assert echo_index.fuzzy_candidates(index, "data pipeline") == []
def test_aliases_in_frontmatter_flow_and_block() -> None:
flow = '---\ntype: project\naliases: ["echo memory", "echo plugin"]\n---\n# x\n'
assert echo_index.aliases_in_frontmatter(flow) == ["echo memory", "echo plugin"]
block = "---\ntype: project\naliases:\n - echo memory\n - echo plugin\n---\n# x\n"
assert echo_index.aliases_in_frontmatter(block) == ["echo memory", "echo plugin"]
assert echo_index.aliases_in_frontmatter("# no frontmatter\n") == []
def test_links_add_related_is_idempotent_and_bidirectional_token() -> None: def test_links_add_related_is_idempotent_and_bidirectional_token() -> None:
text = "---\ntype: person\n---\n\n# Bob\n\n## Related\n" text = "---\ntype: person\n---\n\n# Bob\n\n## Related\n"
new, changed = echo_links.add_related(text, "resources/companies/mpm.md") new, changed = echo_links.add_related(text, "resources/companies/mpm.md")
echo-memory.plugin.src/skills/echo-memory/scripts/vault_lint.py
+11 -9
View File
@@ -143,6 +143,11 @@ def main() -> int:
flag("routing-manifest", f"could not load routing.json ({exc}) — path checks skipped") flag("routing-manifest", f"could not load routing.json ({exc}) — path checks skipped")
all_files = list(walk()) all_files = list(walk())
md_files = [p for p in all_files if p.endswith(".md")]
md_set = set(md_files)
# Fetch every markdown note ONCE, concurrently; all per-note checks below read from
# this shared cache instead of issuing a fresh GET per file per section.
texts = echo.read_many(md_files)
# Path membership + retired-path detection # Path membership + retired-path detection
for path in all_files: for path in all_files:
@@ -159,7 +164,7 @@ def main() -> int:
base = path.rsplit("/", 1)[-1] base = path.rsplit("/", 1)[-1]
if base in SKIP or template_re.search(path) or not path.endswith(".md"): if base in SKIP or template_re.search(path) or not path.endswith(".md"):
continue continue
text = get(path) or "" text = texts.get(path) or ""
raw, fm = parse_frontmatter(text) raw, fm = parse_frontmatter(text)
if "[[" in raw: if "[[" in raw:
flag("frontmatter-wikilink", f"{path}: '[[...]]' inside frontmatter") flag("frontmatter-wikilink", f"{path}: '[[...]]' inside frontmatter")
@@ -184,7 +189,7 @@ def main() -> int:
continue continue
slug = filename[:-3] slug = filename[:-3]
slug_homes.setdefault(slug, []).append(lifecycle) slug_homes.setdefault(slug, []).append(lifecycle)
text = get(f"projects/{lifecycle}/{filename}") or "" text = texts.get(f"projects/{lifecycle}/{filename}") or ""
_, fm = parse_frontmatter(text) _, fm = parse_frontmatter(text)
status = str(fm.get("status", "")).strip() status = str(fm.get("status", "")).strip()
if status and status != lifecycle: if status and status != lifecycle:
@@ -200,7 +205,7 @@ def main() -> int:
# Daily notes: duplicate "## Agent Log" headings # Daily notes: duplicate "## Agent Log" headings
for path in all_files: for path in all_files:
if re.match(r"^journal/daily/.*\.md$", path): if re.match(r"^journal/daily/.*\.md$", path):
text = get(path) or "" text = texts.get(path) or ""
count = len(re.findall(r"(?m)^## Agent Log\s*$", text)) count = len(re.findall(r"(?m)^## Agent Log\s*$", text))
if count > 1: if count > 1:
flag("duplicate-agent-log", f"{path}: {count} '## Agent Log' headings") flag("duplicate-agent-log", f"{path}: {count} '## Agent Log' headings")
@@ -232,8 +237,7 @@ def main() -> int:
f"scope set {scope_updated}; {len(since)} session(s) logged since without a switch — confirm it still reflects current work (or run `echo.py scope set`)") f"scope set {scope_updated}; {len(since)} session(s) logged since without a switch — confirm it still reflects current work (or run `echo.py scope set`)")
# ---- Graph health: broken wikilinks, orphans, index drift ---------------- # ---- Graph health: broken wikilinks, orphans, index drift ----------------
md_files = [p for p in all_files if p.endswith(".md")] # (md_files / md_set were built up top; reuse the shared `texts` cache.)
md_set = set(md_files)
basemap: dict[str, str] = {} basemap: dict[str, str] = {}
for p in md_files: for p in md_files:
basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p) basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)
@@ -248,13 +252,11 @@ def main() -> int:
return cand if cand in md_set else None return cand if cand in md_set else None
return basemap.get(idx_mod.slugify(target)) return basemap.get(idx_mod.slugify(target))
text_cache: dict[str, str] = {}
inbound: set[str] = set() inbound: set[str] = set()
for path in md_files: for path in md_files:
if path.rsplit("/", 1)[-1] in SKIP or skip_link.search(path): if path.rsplit("/", 1)[-1] in SKIP or skip_link.search(path):
continue continue
text = get(path) or "" text = texts.get(path) or ""
text_cache[path] = text
_, body = split_frontmatter(text) _, body = split_frontmatter(text)
for t in links.all_wikilinks(body): for t in links.all_wikilinks(body):
tp = resolve_link(t) tp = resolve_link(t)
@@ -273,7 +275,7 @@ def main() -> int:
for path in md_files: for path in md_files:
if not is_indexable(path) or path in inbound: if not is_indexable(path) or path in inbound:
continue continue
text = text_cache.get(path, get(path) or "") text = texts.get(path) or ""
if not links.related_targets(text): if not links.related_targets(text):
flag("orphan", f"{path}: no inbound links and empty ## Related ({idx_mod.kind_for_path(path)}) — link it for recall") flag("orphan", f"{path}: no inbound links and empty ## Related ({idx_mod.kind_for_path(path)}) — link it for recall")