echo/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py

#!/usr/bin/env python3
"""sweep.py — bring an existing ECHO vault up to the current feature spec.

After upgrading the plugin (entity index, cross-linking, recall), run this once to
backfill what older vaults don't have yet:

  1. ensure the `_agent/index/` folder exists,
  2. **(re)build the entity index** (`_agent/index/entities.json`) from the notes
     already in the vault (people, companies, projects, concepts, references,
     meetings, areas, decisions, semantic/episodic memory, skills),
  3. **backfill incomplete entity frontmatter** — fill a missing/empty `status:` with
     the kind default and seed a missing/empty `tags:` with the note's kind, per the
     KIND_REQUIRED_FM/KIND_STATUS maps in echo_index (what `/echo-health` flags as
     `incomplete-frontmatter`),
  4. **symmetrize cross-links** — for every `## Related` link A -> B, ensure the
     reciprocal B -> A exists (it only adds the missing direction; it never invents
     new links from body text), and
  5. stamp the marker `schema_version` to the current schema (4).

Dry-run by default; pass --apply to write. READ-ONLY without --apply.

**--fast (2.3)** is the incremental mode: it walks the listing (cheap), then fetches
only what's new, gone, missing a content hash, or in today's rotating verification
shard (hash(path) % 7 == weekday — the whole vault gets verified over a week of
fast sweeps while each run stays tiny; --all-shards forces everything). It updates
the LOCAL recall index and the entity index (both machine-owned), so it needs no
--apply gate and writes no notes. `load` auto-runs it when the last fast sweep is
older than ECHO_FAST_SWEEP_DAYS (default 7). This is what keeps the indexes honest
against edits made directly in Obsidian or by other clients.

Cross-platform: pure Python via echo.py.

Usage:  sweep.py [--apply] | sweep.py --fast [--all-shards]
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

import echo          # noqa: E402
import echo_index as idx_mod   # noqa: E402
import echo_links as links     # noqa: E402
import echo_recall             # noqa: E402

CURRENT_SCHEMA = 4
TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)")
SKIP_BASENAMES = {"README.md"}


kind_for = idx_mod.kind_for_path


def fm_block(text: str) -> str:
    if not text.startswith("---"):
        return ""
    end = text.find("\n---", 3)
    return text[:end] if end != -1 else text


def fm_populated(text: str, field: str) -> bool:
    """True when the frontmatter carries a real value for `field` (flow list `[x]`,
    block list items, or a non-empty scalar). Mirrors vault_lint.fm_field_populated."""
    fm = fm_block(text)
    m = re.search(rf"(?m)^{re.escape(field)}:[ \t]*(.*)$", fm)
    if not m:
        return False
    val = m.group(1).strip()
    if val == "[]":
        return False
    if val:
        return True
    rest = fm[m.end():].lstrip("\n")
    first = rest.splitlines()[0] if rest else ""
    return bool(re.match(r"^\s+-\s*\S", first))


def get(path: str) -> str | None:
    status, body = echo.request("GET", echo.vault_url(path))
    if status == 404:
        return None
    echo.check(status, body, f"get {path}")
    return body.decode(errors="replace")


def list_dir(path: str):
    p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/")
    body = get(p)
    if body is None:
        return [], []
    try:
        j = json.loads(body)
    except json.JSONDecodeError:
        return [], []
    entries = list(j.get("files", [])) + list(j.get("folders", []))
    files = [e for e in entries if not e.endswith("/")]
    folders = [e[:-1] for e in entries if e.endswith("/")]
    return files, folders


def walk(prefix: str = ""):
    files, folders = list_dir(prefix)
    for f in files:
        yield prefix + f
    for d in folders:
        yield from walk(f"{prefix}{d}/")


def _stamp_path() -> Path:
    import echo_queue
    return echo_queue.state_dir() / "fast-sweep.stamp"


def _stamp_fast_sweep() -> None:
    try:
        p = _stamp_path()
        p.parent.mkdir(parents=True, exist_ok=True)
        p.write_text(echo.now_iso() + "\n", encoding="utf-8")
    except OSError:
        pass


def fast_sweep_age_days() -> float | None:
    """Days since the last fast/full sweep on THIS machine, or None if never."""
    import datetime as dt
    try:
        stamp = _stamp_path().read_text(encoding="utf-8").strip()
        then = dt.datetime.strptime(stamp, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc)
        return (dt.datetime.now(dt.timezone.utc) - then).total_seconds() / 86400
    except (OSError, ValueError):
        return None


def fast(all_shards: bool = False) -> str:
    """Incremental index maintenance (2.3): true up the LOCAL recall index and the
    entity index against the vault, fetching only new / gone / hash-missing notes
    plus today's rotating verification shard. Index-only — writes no notes; the
    indexes are machine-owned, so no --apply gate. Returns a one-line summary."""
    import datetime as dt
    import echo_recall

    if get("_agent/echo-vault.md") is None:
        return "fast-sweep skipped: vault not bootstrapped"

    listing = [p for p in walk()
               if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
               and not TEMPLATE_RE.search(p)]
    lset = set(listing)

    rix = echo_recall._load_local()
    if rix is None or not rix.n_docs:
        # Nothing to maintain incrementally — build the local index outright
        # (sub-second on a few-hundred-note vault; no vault snapshot write).
        rix = echo_recall.rebuild()
        _stamp_fast_sweep()
        return f"fast-sweep: no local index — built one ({rix.n_docs} docs)"

    index = idx_mod.load()
    ents = index.get("entities", {})
    removed = 0
    for slug in [s for s, e in list(ents.items()) if e.get("path") not in lset]:
        del ents[slug]
        removed += 1
    for p in [p for p in list(rix.length) if p not in lset]:
        rix.remove(p)
        removed += 1

    try:
        weekday = dt.date.fromisoformat(echo.today()).weekday()
    except ValueError:
        weekday = dt.date.today().weekday()

    def in_shard(p: str) -> bool:
        return int(idx_mod.content_hash(p), 16) % 7 == weekday

    cands: set[str] = set()
    for p in listing:
        if not echo_recall._indexable(p):
            continue
        if p not in rix.length:            # new note (any client, any tool)
            cands.add(p)
            continue
        kind = kind_for(p)
        if kind:
            slug = idx_mod.slugify(p.rsplit("/", 1)[-1][:-3])
            if not (ents.get(slug) or {}).get("h"):   # pre-2.3 entry: hash it once
                cands.add(p)
                continue
        if all_shards or in_shard(p):      # rotating verification shard
            cands.add(p)

    texts = echo.read_many(sorted(cands))
    changed = 0
    ents_dirty = removed > 0
    for p, t in texts.items():
        if t is None:
            continue
        h = idx_mod.content_hash(t)
        if rix.content_hash(p) != h:
            rix.add(p, t)
            changed += 1
        kind = kind_for(p)
        if kind:
            slug = idx_mod.slugify(p.rsplit("/", 1)[-1][:-3])
            prev = ents.get(slug) or {}
            if prev.get("h") != h or prev.get("path") != p:
                title = links.first_h1(t) or p.rsplit("/", 1)[-1][:-3]
                aliases = list(prev.get("aliases", [])) + idx_mod.aliases_in_frontmatter(t)
                idx_mod.upsert(index, slug, p, kind, title, aliases, h=h)
                ents_dirty = True

    echo_recall.save_local(rix)
    if ents_dirty:
        idx_mod.save(index)
    _stamp_fast_sweep()
    return (f"fast-sweep: {len(cands)} note(s) checked, {changed} reindexed, "
            f"{removed} removed" + (" (all shards)" if all_shards else ""))


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Bring an ECHO vault up to the current feature spec")
    ap.add_argument("--apply", action="store_true")
    ap.add_argument("--fast", action="store_true",
                    help="incremental index maintenance (local recall + entity index only)")
    ap.add_argument("--all-shards", action="store_true",
                    help="with --fast: verify every indexed note, not just today's shard")
    args = ap.parse_args(argv)
    if args.fast:
        print(fast(all_shards=args.all_shards))
        return 0
    apply = args.apply
    tag = "APPLY" if apply else "PLAN "

    if get("_agent/echo-vault.md") is None:
        print("sweep: marker missing — vault not bootstrapped. Run bootstrap.py first.", file=sys.stderr)
        return 3

    print(f"sweep: {'applying changes' if apply else 'dry-run (no writes)'}\n")

    # ---- 1. index folder ------------------------------------------------------
    if get("_agent/index/README.md") is None:
        print(f"sweep: {tag} create _agent/index/README.md")
        if apply:
            echo.request("PUT", echo.vault_url("_agent/index/README.md"),
                         data=b"# index\n\nMachine-maintained entity index. See the echo-memory plugin.\n",
                         headers={"Content-Type": "text/markdown"})

    all_files = [p for p in walk()
                 if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
                 and not TEMPLATE_RE.search(p)]

    # Fetch every note's content ONCE, concurrently. All three passes below (index,
    # recall, link symmetrize) read from this shared cache, so no file is fetched
    # twice and link targets aren't re-fetched per reference.
    texts = echo.read_many(all_files)
    print(f"sweep: read {len(texts)} notes (concurrent, keep-alive)\n")

    # ---- 2. (re)build entity index -------------------------------------------
    index = idx_mod.load()
    existing = dict(index.get("entities", {}))
    rebuilt = idx_mod.empty_index()
    added = updated = 0
    for path in all_files:
        kind = kind_for(path)
        if kind is None:
            continue
        base = path.rsplit("/", 1)[-1][:-3]
        slug = idx_mod.slugify(base)
        text = texts.get(path) or ""
        title = links.first_h1(text) or base
        prev = existing.get(slug)
        # Frontmatter aliases are authoritative and re-folded every rebuild; prior index
        # aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
        prev_aliases = prev.get("aliases", []) if prev else []
        aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
        idx_mod.upsert(rebuilt, slug, path, kind, title, aliases,
                       h=idx_mod.content_hash(text))
        if not prev:
            added += 1
        elif prev.get("path") != path or prev.get("title") != title:
            updated += 1
    print(f"sweep: {tag} entity index — {len(rebuilt['entities'])} entities "
          f"({added} new, {updated} changed)")
    if apply:
        idx_mod.save(rebuilt)

    # ---- 2b. (re)build the recall (BM25) index -------------------------------
    if apply:
        rix = echo_recall.rebuild(prefetched=texts, snapshot=True)
        _stamp_fast_sweep()
        print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25; local + vault snapshot)")
    else:
        n_idx = sum(1 for p in all_files if echo_recall._indexable(p))
        print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} corpus notes "
              f"(entities + sessions/journal)")

    # ---- 3. backfill incomplete entity frontmatter ----------------------------
    # What /echo-health flags as incomplete-frontmatter, filled mechanically: a missing
    # or empty status gets the kind default; missing/empty tags get the kind as a
    # baseline tag. cmd_fm is create-or-replace, so absent keys are inserted surgically.
    fixes: list[tuple[str, str, str]] = []
    for path in all_files:
        kind = kind_for(path)
        if kind is None:
            continue
        text = texts.get(path) or ""
        if not text.startswith("---"):
            continue  # no frontmatter block at all — lint flags it; not a mechanical fill
        for field in idx_mod.KIND_REQUIRED_FM.get(kind, ()):
            if not fm_populated(text, field):
                value = (idx_mod.KIND_STATUS.get(kind, "active") if field == "status"
                         else json.dumps([kind]))
                fixes.append((path, field, value))
    print(f"sweep: {tag} frontmatter backfill — {len(fixes)} field(s) to fill")
    for path, field, value in fixes[:40]:
        print(f"          {path}: {field} -> {value}")
    if len(fixes) > 40:
        print(f"          ... and {len(fixes) - 40} more")
    if apply:
        for path, field, value in fixes:
            echo.cmd_fm(path, field, value)

    # ---- 4. symmetrize existing Related links --------------------------------
    fileset = set(all_files)
    basemap: dict[str, str] = {}
    for p in all_files:
        basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)

    def resolve_target(t: str) -> str | None:
        t = t.strip()
        if "/" in t:
            cand = t if t.endswith(".md") else t + ".md"
            return cand if cand in fileset else None
        return basemap.get(idx_mod.slugify(t))

    missing = set()  # (target_path, source_path) — reciprocal link to add on target
    for path in all_files:
        text = texts.get(path)
        if text is None:
            continue
        for tgt in links.related_targets(text):
            tp = resolve_target(tgt)
            if not tp or tp == path:
                continue
            back = {resolve_target(t) for t in links.related_targets(texts.get(tp) or "")}
            if path not in back:
                missing.add((tp, path))
    missing = sorted(missing)
    print(f"sweep: {tag} reciprocal links — {len(missing)} to add")
    for tp, sp in missing[:40]:
        print(f"          {tp}  +=  [[{links.link_token(sp)}]]")
    if len(missing) > 40:
        print(f"          ... and {len(missing) - 40} more")
    if apply:
        for tp, sp in missing:
            links.add_one(tp, sp)

    # ---- 5. stamp schema ------------------------------------------------------
    marker = get("_agent/echo-vault.md") or ""
    cur = next((int(m) for m in re.findall(r"(?m)^schema_version:\s*(\d+)", marker)), 0)
    if cur < CURRENT_SCHEMA:
        print(f"sweep: {tag} schema_version {cur} -> {CURRENT_SCHEMA}")
        if apply:
            echo.cmd_fm("_agent/echo-vault.md", "schema_version", str(CURRENT_SCHEMA))

    print(f"\nsweep: {'done — run vault_lint.py to confirm invariants' if apply else 'dry-run complete. Re-run with --apply to write.'}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())