2.3.0 — the index train: local-first recall index, incremental sweep, stemming + alias expansion

One schema-bump release (recall-index schema 3, entity-index schema 2; old
recall indexes rebuild automatically, no vault migration):

- Local-first recall index: the live BM25 index lives in the machine state dir
  (keyed by endpoint hash); capture's upkeep is a zero-network atomic file
  write, no advisory lock — replacing the O(vault) GET/PUT-whole-index round
  trip per write. The vault copy is a snapshot (sweep + session-end) used to
  seed fresh machines / catch up after another client swept
  (ECHO_RECALL_SYNC_HOURS, default 24). The read path never PUTs the vault.
- Incremental sweep: entity + recall meta carry 16-char content hashes;
  `sweep --fast` fetches only new/gone/hash-missing notes plus a rotating
  weekday shard (--all-shards forces everything); deletions drop from both
  indexes; index-only so no --apply gate. `load` auto-runs it past
  ECHO_FAST_SWEEP_DAYS (7); doctor reports index freshness. Obsidian-side
  edits now reach the indexes without manual maintenance.
- Stemming + alias expansion: echo_stem.py (conservative Porter-lite, unit-
  tested families + over-stemming guards) applied at index AND query time;
  a query fuzzy-matching an entity folds its title/alias vocabulary into the
  BM25 query at half weight — expansion can only boost docs containing the
  terms. capture hashes the note's FINAL content (auto-link reordered first).

Eval gold set 8 -> 14 queries (6 paraphrases): recall@5/MRR 1.00/1.00 vs
keyword baseline 0.75/0.79. +3 unit tests, +10 e2e; test harnesses now isolate
ECHO_STATE_DIR. All seven suites green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py

@@ -18,9 +18,19 @@ backfill what older vaults don't have yet:
   5. stamp the marker `schema_version` to the current schema (4).
 
 Dry-run by default; pass --apply to write. READ-ONLY without --apply.
+
+**--fast (2.3)** is the incremental mode: it walks the listing (cheap), then fetches
+only what's new, gone, missing a content hash, or in today's rotating verification
+shard (hash(path) % 7 == weekday — the whole vault gets verified over a week of
+fast sweeps while each run stays tiny; --all-shards forces everything). It updates
+the LOCAL recall index and the entity index (both machine-owned), so it needs no
+--apply gate and writes no notes. `load` auto-runs it when the last fast sweep is
+older than ECHO_FAST_SWEEP_DAYS (default 7). This is what keeps the indexes honest
+against edits made directly in Obsidian or by other clients.
+
 Cross-platform: pure Python via echo.py.
 
-Usage:  sweep.py [--apply]
+Usage:  sweep.py [--apply] | sweep.py --fast [--all-shards]
 """
 
 from __future__ import annotations
@@ -102,10 +112,128 @@ def walk(prefix: str = ""):
         yield from walk(f"{prefix}{d}/")
 
 
+def _stamp_path() -> Path:
+    import echo_queue
+    return echo_queue.state_dir() / "fast-sweep.stamp"
+
+
+def _stamp_fast_sweep() -> None:
+    try:
+        p = _stamp_path()
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(echo.now_iso() + "\n", encoding="utf-8")
+    except OSError:
+        pass
+
+
+def fast_sweep_age_days() -> float | None:
+    """Days since the last fast/full sweep on THIS machine, or None if never."""
+    import datetime as dt
+    try:
+        stamp = _stamp_path().read_text(encoding="utf-8").strip()
+        then = dt.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc)
+        return (dt.datetime.now(dt.timezone.utc) - then).total_seconds() / 86400
+    except (OSError, ValueError):
+        return None
+
+
+def fast(all_shards: bool = False) -> str:
+    """Incremental index maintenance (2.3): true up the LOCAL recall index and the
+    entity index against the vault, fetching only new / gone / hash-missing notes
+    plus today's rotating verification shard. Index-only — writes no notes; the
+    indexes are machine-owned, so no --apply gate. Returns a one-line summary."""
+    import datetime as dt
+    import echo_recall
+
+    if get("_agent/echo-vault.md") is None:
+        return "fast-sweep skipped: vault not bootstrapped"
+
+    listing = [p for p in walk()
+               if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
+               and not TEMPLATE_RE.search(p)]
+    lset = set(listing)
+
+    rix = echo_recall._load_local()
+    if rix is None or not rix.n_docs:
+        # Nothing to maintain incrementally — build the local index outright
+        # (sub-second on a few-hundred-note vault; no vault snapshot write).
+        rix = echo_recall.rebuild()
+        _stamp_fast_sweep()
+        return f"fast-sweep: no local index — built one ({rix.n_docs} docs)"
+
+    index = idx_mod.load()
+    ents = index.get("entities", {})
+    removed = 0
+    for slug in [s for s, e in list(ents.items()) if e.get("path") not in lset]:
+        del ents[slug]
+        removed += 1
+    for p in [p for p in list(rix.length) if p not in lset]:
+        rix.remove(p)
+        removed += 1
+
+    try:
+        weekday = dt.date.fromisoformat(echo.today()).weekday()
+    except ValueError:
+        weekday = dt.date.today().weekday()
+
+    def in_shard(p: str) -> bool:
+        return int(idx_mod.content_hash(p), 16) % 7 == weekday
+
+    cands: set[str] = set()
+    for p in listing:
+        if not echo_recall._indexable(p):
+            continue
+        if p not in rix.length:            # new note (any client, any tool)
+            cands.add(p)
+            continue
+        kind = kind_for(p)
+        if kind:
+            slug = idx_mod.slugify(p.rsplit("/", 1)[-1][:-3])
+            if not (ents.get(slug) or {}).get("h"):   # pre-2.3 entry: hash it once
+                cands.add(p)
+                continue
+        if all_shards or in_shard(p):      # rotating verification shard
+            cands.add(p)
+
+    texts = echo.read_many(sorted(cands))
+    changed = 0
+    ents_dirty = removed > 0
+    for p, t in texts.items():
+        if t is None:
+            continue
+        h = idx_mod.content_hash(t)
+        if rix.content_hash(p) != h:
+            rix.add(p, t)
+            changed += 1
+        kind = kind_for(p)
+        if kind:
+            slug = idx_mod.slugify(p.rsplit("/", 1)[-1][:-3])
+            prev = ents.get(slug) or {}
+            if prev.get("h") != h or prev.get("path") != p:
+                title = links.first_h1(t) or p.rsplit("/", 1)[-1][:-3]
+                aliases = list(prev.get("aliases", [])) + idx_mod.aliases_in_frontmatter(t)
+                idx_mod.upsert(index, slug, p, kind, title, aliases, h=h)
+                ents_dirty = True
+
+    echo_recall.save_local(rix)
+    if ents_dirty:
+        idx_mod.save(index)
+    _stamp_fast_sweep()
+    return (f"fast-sweep: {len(cands)} note(s) checked, {changed} reindexed, "
+            f"{removed} removed" + (" (all shards)" if all_shards else ""))
+
+
 def main(argv: list[str] | None = None) -> int:
     ap = argparse.ArgumentParser(description="Bring an ECHO vault up to the current feature spec")
     ap.add_argument("--apply", action="store_true")
+    ap.add_argument("--fast", action="store_true",
+                    help="incremental index maintenance (local recall + entity index only)")
+    ap.add_argument("--all-shards", action="store_true",
+                    help="with --fast: verify every indexed note, not just today's shard")
     args = ap.parse_args(argv)
+    if args.fast:
+        print(fast(all_shards=args.all_shards))
+        return 0
     apply = args.apply
     tag = "APPLY" if apply else "PLAN "
 
@@ -151,7 +279,8 @@ def main(argv: list[str] | None = None) -> int:
         # aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
         prev_aliases = prev.get("aliases", []) if prev else []
         aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
-        idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
+        idx_mod.upsert(rebuilt, slug, path, kind, title, aliases,
+                       h=idx_mod.content_hash(text))
         if not prev:
             added += 1
         elif prev.get("path") != path or prev.get("title") != title:
@@ -163,8 +292,9 @@ def main(argv: list[str] | None = None) -> int:
 
     # ---- 2b. (re)build the recall (BM25) index -------------------------------
     if apply:
-        rix = echo_recall.rebuild(prefetched=texts)
-        print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)")
+        rix = echo_recall.rebuild(prefetched=texts, snapshot=True)
+        _stamp_fast_sweep()
+        print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25; local + vault snapshot)")
     else:
         n_idx = sum(1 for p in all_files if echo_recall._indexable(p))
         print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} corpus notes "