echo/echo-memory.plugin.src/skills/echo-memory/scripts/sweep.py

#!/usr/bin/env python3
"""sweep.py — bring an existing ECHO vault up to the current feature spec.

After upgrading the plugin (entity index, cross-linking, recall), run this once to
backfill what older vaults don't have yet:

  1. ensure the `_agent/index/` folder exists,
  2. **(re)build the entity index** (`_agent/index/entities.json`) from the notes
     already in the vault (people, companies, projects, concepts, references,
     meetings, areas, decisions, semantic/episodic memory, skills),
  3. **backfill incomplete entity frontmatter** — fill a missing/empty `status:` with
     the kind default and seed a missing/empty `tags:` with the note's kind, per the
     KIND_REQUIRED_FM/KIND_STATUS maps in echo_index (what `/echo-health` flags as
     `incomplete-frontmatter`),
  4. **symmetrize cross-links** — for every `## Related` link A -> B, ensure the
     reciprocal B -> A exists (it only adds the missing direction; it never invents
     new links from body text), and
  5. stamp the marker `schema_version` to the current schema (4).

Dry-run by default; pass --apply to write. READ-ONLY without --apply.
Cross-platform: pure Python via echo.py.

Usage:  sweep.py [--apply]
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

import echo          # noqa: E402
import echo_index as idx_mod   # noqa: E402
import echo_links as links     # noqa: E402
import echo_recall             # noqa: E402

CURRENT_SCHEMA = 4
TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)")
SKIP_BASENAMES = {"README.md"}


kind_for = idx_mod.kind_for_path


def fm_block(text: str) -> str:
    if not text.startswith("---"):
        return ""
    end = text.find("\n---", 3)
    return text[:end] if end != -1 else text


def fm_populated(text: str, field: str) -> bool:
    """True when the frontmatter carries a real value for `field` (flow list `[x]`,
    block list items, or a non-empty scalar). Mirrors vault_lint.fm_field_populated."""
    fm = fm_block(text)
    m = re.search(rf"(?m)^{re.escape(field)}:[ \t]*(.*)$", fm)
    if not m:
        return False
    val = m.group(1).strip()
    if val == "[]":
        return False
    if val:
        return True
    rest = fm[m.end():].lstrip("\n")
    first = rest.splitlines()[0] if rest else ""
    return bool(re.match(r"^\s+-\s*\S", first))


def get(path: str) -> str | None:
    status, body = echo.request("GET", echo.vault_url(path))
    if status == 404:
        return None
    echo.check(status, body, f"get {path}")
    return body.decode(errors="replace")


def list_dir(path: str):
    p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/")
    body = get(p)
    if body is None:
        return [], []
    try:
        j = json.loads(body)
    except json.JSONDecodeError:
        return [], []
    entries = list(j.get("files", [])) + list(j.get("folders", []))
    files = [e for e in entries if not e.endswith("/")]
    folders = [e[:-1] for e in entries if e.endswith("/")]
    return files, folders


def walk(prefix: str = ""):
    files, folders = list_dir(prefix)
    for f in files:
        yield prefix + f
    for d in folders:
        yield from walk(f"{prefix}{d}/")


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Bring an ECHO vault up to the current feature spec")
    ap.add_argument("--apply", action="store_true")
    args = ap.parse_args(argv)
    apply = args.apply
    tag = "APPLY" if apply else "PLAN "

    if get("_agent/echo-vault.md") is None:
        print("sweep: marker missing — vault not bootstrapped. Run bootstrap.py first.", file=sys.stderr)
        return 3

    print(f"sweep: {'applying changes' if apply else 'dry-run (no writes)'}\n")

    # ---- 1. index folder ------------------------------------------------------
    if get("_agent/index/README.md") is None:
        print(f"sweep: {tag} create _agent/index/README.md")
        if apply:
            echo.request("PUT", echo.vault_url("_agent/index/README.md"),
                         data=b"# index\n\nMachine-maintained entity index. See the echo-memory plugin.\n",
                         headers={"Content-Type": "text/markdown"})

    all_files = [p for p in walk()
                 if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
                 and not TEMPLATE_RE.search(p)]

    # Fetch every note's content ONCE, concurrently. All three passes below (index,
    # recall, link symmetrize) read from this shared cache, so no file is fetched
    # twice and link targets aren't re-fetched per reference.
    texts = echo.read_many(all_files)
    print(f"sweep: read {len(texts)} notes (concurrent, keep-alive)\n")

    # ---- 2. (re)build entity index -------------------------------------------
    index = idx_mod.load()
    existing = dict(index.get("entities", {}))
    rebuilt = idx_mod.empty_index()
    added = updated = 0
    for path in all_files:
        kind = kind_for(path)
        if kind is None:
            continue
        base = path.rsplit("/", 1)[-1][:-3]
        slug = idx_mod.slugify(base)
        text = texts.get(path) or ""
        title = links.first_h1(text) or base
        prev = existing.get(slug)
        # Frontmatter aliases are authoritative and re-folded every rebuild; prior index
        # aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
        prev_aliases = prev.get("aliases", []) if prev else []
        aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
        idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
        if not prev:
            added += 1
        elif prev.get("path") != path or prev.get("title") != title:
            updated += 1
    print(f"sweep: {tag} entity index — {len(rebuilt['entities'])} entities "
          f"({added} new, {updated} changed)")
    if apply:
        idx_mod.save(rebuilt)

    # ---- 2b. (re)build the recall (BM25) index -------------------------------
    if apply:
        rix = echo_recall.rebuild(prefetched=texts)
        print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)")
    else:
        n_idx = sum(1 for p in all_files if echo_recall._indexable(p))
        print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} corpus notes "
              f"(entities + sessions/journal)")

    # ---- 3. backfill incomplete entity frontmatter ----------------------------
    # What /echo-health flags as incomplete-frontmatter, filled mechanically: a missing
    # or empty status gets the kind default; missing/empty tags get the kind as a
    # baseline tag. cmd_fm is create-or-replace, so absent keys are inserted surgically.
    fixes: list[tuple[str, str, str]] = []
    for path in all_files:
        kind = kind_for(path)
        if kind is None:
            continue
        text = texts.get(path) or ""
        if not text.startswith("---"):
            continue  # no frontmatter block at all — lint flags it; not a mechanical fill
        for field in idx_mod.KIND_REQUIRED_FM.get(kind, ()):
            if not fm_populated(text, field):
                value = (idx_mod.KIND_STATUS.get(kind, "active") if field == "status"
                         else json.dumps([kind]))
                fixes.append((path, field, value))
    print(f"sweep: {tag} frontmatter backfill — {len(fixes)} field(s) to fill")
    for path, field, value in fixes[:40]:
        print(f"          {path}: {field} -> {value}")
    if len(fixes) > 40:
        print(f"          ... and {len(fixes) - 40} more")
    if apply:
        for path, field, value in fixes:
            echo.cmd_fm(path, field, value)

    # ---- 4. symmetrize existing Related links --------------------------------
    fileset = set(all_files)
    basemap: dict[str, str] = {}
    for p in all_files:
        basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)

    def resolve_target(t: str) -> str | None:
        t = t.strip()
        if "/" in t:
            cand = t if t.endswith(".md") else t + ".md"
            return cand if cand in fileset else None
        return basemap.get(idx_mod.slugify(t))

    missing = set()  # (target_path, source_path) — reciprocal link to add on target
    for path in all_files:
        text = texts.get(path)
        if text is None:
            continue
        for tgt in links.related_targets(text):
            tp = resolve_target(tgt)
            if not tp or tp == path:
                continue
            back = {resolve_target(t) for t in links.related_targets(texts.get(tp) or "")}
            if path not in back:
                missing.add((tp, path))
    missing = sorted(missing)
    print(f"sweep: {tag} reciprocal links — {len(missing)} to add")
    for tp, sp in missing[:40]:
        print(f"          {tp}  +=  [[{links.link_token(sp)}]]")
    if len(missing) > 40:
        print(f"          ... and {len(missing) - 40} more")
    if apply:
        for tp, sp in missing:
            links.add_one(tp, sp)

    # ---- 5. stamp schema ------------------------------------------------------
    marker = get("_agent/echo-vault.md") or ""
    cur = next((int(m) for m in re.findall(r"(?m)^schema_version:\s*(\d+)", marker)), 0)
    if cur < CURRENT_SCHEMA:
        print(f"sweep: {tag} schema_version {cur} -> {CURRENT_SCHEMA}")
        if apply:
            echo.cmd_fm("_agent/echo-vault.md", "schema_version", str(CURRENT_SCHEMA))

    print(f"\nsweep: {'done — run vault_lint.py to confirm invariants' if apply else 'dry-run complete. Re-run with --apply to write.'}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())