chorus/echo-memory.plugin.src/skills/echo-memory/scripts/echo_ops.py

#!/usr/bin/env python3
"""echo_ops.py — high-level ECHO operations layered on the client + index + links.

  resolve  — map a mention to its canonical note path (or where to create it)
  recall   — search, then expand one hop along links/source_notes for connected context
  link     — create bidirectional `## Related` links between two notes
  capture  — one-call write: route + canonical frontmatter + index + auto-link + agent-log

Network goes through echo.py; routing/aliases through echo_index; links through echo_links.
"""

from __future__ import annotations

import json
import os
import re
import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))
import echo          # noqa: E402
import echo_index as idx_mod   # noqa: E402
import echo_links as links     # noqa: E402

# A fuzzy candidate scoring at/above this blocks `capture` from creating what is very
# likely a duplicate (require --force to create anyway, or --merge-into to update the
# existing entity). Below the gate, candidates are surfaced as a warning only.
DUP_GATE = float(os.environ.get("ECHO_DUP_GATE", "0.6"))

# Existing-entity capture appends a dated bullet under this per-kind heading.
LOG_HEADING = {
    "project": "Session History", "person": "Log", "company": "Log",
    "semantic": "Observations", "area": "Log", "concept": "Notes",
    "reference": "Notes", "meeting": "Notes", "decision": "Notes",
    "skill": "Notes", "episodic": "Notes", "working": "Notes",
}


# ---------------------------------------------------------------- resolve -----
def resolve(mention: str) -> int:
    index = idx_mod.load()
    slug, e = idx_mod.resolve(index, mention)
    if e:
        print(json.dumps({"match": True, "slug": slug, **e}, ensure_ascii=False, indent=2))
        return 0
    # No exact match — surface fuzzy candidates so a shortened/expanded name (e.g. "echo
    # memory" for the project "echo") reveals the existing note instead of looking absent.
    cands = idx_mod.fuzzy_candidates(index, mention)
    out = {"match": False, "mention": mention, "suggest_slug": idx_mod.slugify(mention)}
    if cands:
        out["candidates"] = [{"slug": s, "path": c.get("path"), "title": c.get("title"),
                              "kind": c.get("kind"), "score": sc} for s, c, sc in cands]
        out["note"] = ("no exact match, but similar entities exist (see candidates) — check "
                       "these before creating a new note; reuse the right path or `echo.py link`.")
    else:
        out["note"] = "no index entry — derive a path with the right --kind and create it"
    print(json.dumps(out, ensure_ascii=False, indent=2))
    return 0


# ------------------------------------------------------------------- link -----
def link(a_path: str, b_path: str, as_json: bool = False) -> int:
    a_changed, b_changed = links.link_bidirectional(a_path, b_path)
    if as_json:
        import echo_output
        env = echo_output.envelope("link", {"a": a_path, "b": b_path,
                                            "a_changed": a_changed, "b_changed": b_changed})
        print(json.dumps(env, ensure_ascii=False))
        return 0
    print(f"ok: linked {a_path} <-> {b_path} "
          f"(added: {'A' if a_changed else '-'}{'B' if b_changed else '-'})")
    return 0


# ----------------------------------------------------------------- recall -----
def recall(query, limit: int = 8, as_json: bool = False) -> int:
    """Hybrid lexical (BM25) + graph recall — implemented in echo_recall. Kept here as
    the stable public entrypoint (echo.py and /echo-recall call echo_ops.recall)."""
    import echo_recall  # lazy: only pulls the BM25 modules when recall is actually run
    return echo_recall.recall(query, limit=limit, as_json=as_json)


# ----------------------------------------------------------- agent log -------
def ensure_daily_log(line: str) -> None:
    """Resilient: ensure today's daily note + its `## Agent Log` heading exist, then
    idempotently append the line. Best-effort — never raises into the caller."""
    try:
        date = echo.today()
        path = f"journal/daily/{date}.md"
        status, body = echo.request("GET", echo.vault_url(path))
        if status == 404:
            ts, tb = echo.request("GET", echo.vault_url("journal/templates/daily-note-template.md"))
            tmpl = tb.decode(errors="replace") if ts == 200 else f"# {date}\n\n## Agent Log\n\n## Related\n"
            tmpl = tmpl.replace("{{date:YYYY-MM-DD}}", date).replace("{{DATE}}", date)
            echo.request("PUT", echo.vault_url(path), data=tmpl.encode(),
                         headers={"Content-Type": "text/markdown"})
            text = tmpl
        else:
            text = body.decode(errors="replace")
        if not re.search(r"(?m)^## Agent Log\s*$", text):
            echo.request("POST", echo.vault_url(path), data=b"\n\n## Agent Log\n",
                         headers={"Content-Type": "text/markdown"})
        status, body = echo.request("GET", echo.vault_url(path))
        if status == 200 and line in body.decode(errors="replace"):
            return
        echo.request("PATCH", echo.vault_url(path),
                     data=echo.normalize_patch_body((line + "\n").encode(), "append", "heading"),
                     headers={"Operation": "append", "Target-Type": "heading",
                              "Target": f"{date}::Agent Log", "Content-Type": "text/markdown"})
    except Exception as exc:
        print(f"echo_ops: agent-log skipped ({exc})", file=sys.stderr)


# ----------------------------------------------------------------- capture ---
def _build_note(fm_type: str, status_v: str, today_s: str, title: str,
                body_text: str, sources: list[str], aliases: list[str] | None = None,
                tags: list[str] | None = None) -> str:
    src = "[" + ", ".join(json.dumps(s) for s in sources) + "]"
    fm = ["---", f"type: {fm_type}"]
    if status_v:
        fm.append(f"status: {status_v}")
    # Never scaffold an empty `tags: []` — an entity note is born complete (the caller
    # seeds the kind as a baseline tag), so the incomplete-frontmatter lint stays quiet.
    tags_v = "[" + ", ".join(tags or []) + "]"
    fm += [f"created: {today_s}", f"updated: {today_s}", f"tags: {tags_v}"]
    if aliases:
        # Obsidian-native, durable home for aliases; sweep folds these back into the index.
        fm.append("aliases: [" + ", ".join(json.dumps(a) for a in aliases) + "]")
    fm += ["agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""]
    out = "\n".join(fm)
    if body_text.strip():
        out += body_text.strip() + "\n"
    out += "\n## Related\n"
    return out


def _dated_block(today_s: str, body_text: str) -> tuple[str, str]:
    """Render a capture body as a dated log entry that PRESERVES the whole body:
    the first line rides on the bullet, every further line is indented under it as
    a markdown continuation. Returns (bullet, full_block) — the bullet alone is the
    idempotency key (same first-line-per-day semantics as before)."""
    lines = [ln.rstrip() for ln in body_text.strip().splitlines()]
    bullet = f"- {today_s}: {lines[0] if lines else '(update)'}"
    block = bullet
    for ln in lines[1:]:
        block += "\n" + (f"  {ln}" if ln else "")
    return bullet, block


def _append_to_existing(path: str, kind: str, today_s: str, body_text: str) -> None:
    text = links.get_text(path) or ""
    h1 = links.first_h1(text) or path.rsplit("/", 1)[-1][:-3]
    heading = LOG_HEADING.get(kind, "Notes")
    if not re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", text):
        echo.request("POST", echo.vault_url(path), data=f"\n\n## {heading}\n".encode(),
                     headers={"Content-Type": "text/markdown"})
    bullet, block = _dated_block(today_s, body_text)
    status, body = echo.request("GET", echo.vault_url(path))
    if status == 200 and bullet in body.decode(errors="replace"):
        return
    echo.request("PATCH", echo.vault_url(path),
                 data=echo.normalize_patch_body((block + "\n").encode(), "append", "heading"),
                 headers={"Operation": "append", "Target-Type": "heading",
                          "Target": f"{h1}::{heading}", "Content-Type": "text/markdown"})
    echo.cmd_fm(path, "updated", json.dumps(today_s))


def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = "",
            aliases=None, sources=None, tags=None, date: str | None = None,
            domain: str = "business", inbox: bool = False, no_log: bool = False,
            as_json: bool = False, dry_run: bool = False, force: bool = False,
            merge_into: str | None = None) -> int:
    import contextlib
    import io
    import echo_output
    import echo_quality

    real_stdout = sys.stdout

    def quiet():
        # M4: in --json mode, swallow the helper "ok:" chatter so stdout is clean JSON.
        return contextlib.redirect_stdout(io.StringIO()) if as_json else contextlib.nullcontext()

    def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False,
             near=None) -> int:
        if as_json:
            act = f"dry-run:{action}" if dry else action
            data = {"kind": kind, "path": path, "title": title, "links_added": links}
            if near:
                data["near_duplicates"] = near
            env = echo_output.envelope(act, data, ok=ok)
            print(json.dumps(env, ensure_ascii=False), file=real_stdout)
        elif dry:
            print(f"would {action} {kind or '-'} -> {path}")
        # non-dry human output is the helper "ok:" lines + the summary printed by the caller.
        return 0 if ok else 1

    body_text = echo.read_body(file_arg).decode("utf-8", errors="replace")
    today_s = echo.today()
    aliases = [a.strip() for a in (aliases or []) if a.strip()]
    sources = [s.strip() for s in (sources or []) if s.strip()]
    tags = [t.strip() for t in (tags or []) if t.strip()]

    # Unknown home -> defer to the inbox (a single idempotent capture line).
    if inbox or not kind:
        if dry_run:
            return done("inbox", "inbox/captures/inbox.md", dry=True)
        line = f"- {today_s}: {title}"
        if body_text.strip():
            line += f" — {body_text.strip().splitlines()[0]}"
        with quiet():
            rc = echo.cmd_append("inbox/captures/inbox.md", line)
        return done("inbox", "inbox/captures/inbox.md", ok=rc == 0)

    slug = idx_mod.slugify(title)
    index = idx_mod.load()
    match_slug, existing = idx_mod.resolve(index, title)
    # --merge-into: the operator has already identified the canonical entity (e.g. after
    # a duplicate-gate stop) — route this capture as an UPDATE to it, whatever the title.
    if merge_into and not existing:
        match_slug, existing = idx_mod.resolve(index, merge_into)
        if not existing:
            print(f"echo_ops: --merge-into '{merge_into}' matches no entity in the index "
                  f"(try `resolve` first)", file=sys.stderr)
            return 2
    existing_reachable = bool(existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200)

    # Pre-write duplicate gate: a strong fuzzy candidate means this title is very likely
    # an existing entity under a different name. STOP before creating the duplicate —
    # after the fact, the warning arrives too late (the parallel note already exists).
    # gate_candidates applies the 1.5.1 precision rules (same-kind only; a lone shared
    # token blocks only when unique to that entity) so vault-common words can't gate.
    gate_hits = []
    if not existing_reachable and not force and not dry_run:
        gate_hits = [{"slug": s, "path": c.get("path"), "title": c.get("title"),
                      "kind": c.get("kind"), "score": sc}
                     for s, c, sc in idx_mod.gate_candidates(index, title, kind=kind,
                                                             threshold=DUP_GATE)]
    if gate_hits:
        if as_json:
            env = echo_output.envelope("duplicate-gate", {
                "kind": kind, "title": title, "candidates": gate_hits,
                "note": "likely duplicate — re-run with --merge-into <slug> to update the "
                        "existing entity, or --force to create anyway"}, ok=False)
            print(json.dumps(env, ensure_ascii=False), file=real_stdout)
        else:
            print(f"STOP: '{title}' likely already exists — not creating a duplicate.",
                  file=real_stdout)
            for c in gate_hits:
                print(f"  candidate: {c['slug']} -> {c['path']} (score {c['score']})",
                      file=real_stdout)
            print("  re-run with --merge-into <slug> to update the existing entity, "
                  "or --force to create anyway.", file=real_stdout)
        return 76  # distinct exit: duplicate gate (cf. 75 lock, 78 config)

    if dry_run:
        if existing_reachable:
            return done("update", existing["path"], dry=True)
        s2 = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
        near = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
        plan = done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain),
                    dry=True, near=near)
        if (not as_json and not force
                and idx_mod.gate_candidates(index, title, kind=kind, threshold=DUP_GATE)):
            # (in --json mode the near_duplicates field carries this; keep stdout clean)
            print("note: a real run would STOP at the duplicate gate — use --merge-into "
                  "or --force.", file=real_stdout)
        return plan

    near_dupes: list[str] = []
    index_title = title          # title to record in the index entry
    index_aliases = list(aliases)
    with quiet():
        if existing_reachable:
            # Reuse the matched entity's CANONICAL slug — never re-slug from the mention, or
            # two index slugs end up pointing at one note. Keep its title; learn the mention
            # as an alias when it's a new distinctive form, so this name resolves next time.
            slug = match_slug or slug
            path = existing["path"]
            kind = existing.get("kind", kind)
            index_title = existing.get("title") or title
            known = {idx_mod._norm(a) for a in existing.get("aliases", [])} | {match_slug}
            if idx_mod.slugify(title) not in known and len(title.strip()) >= echo_quality.MIN_NAME_LEN:
                index_aliases.append(title)
            _append_to_existing(path, kind, today_s, body_text)
            action = "updated"
        else:
            if not status_v:
                # Every entity note is born with a status — a kind-appropriate default
                # (KIND_STATUS) instead of the old project-only special case. Missing
                # status was the root cause of the frontmatter-completeness drift.
                status_v = idx_mod.KIND_STATUS.get(kind, "active")
            # Surface (don't silently create alongside) an existing entity this resembles —
            # the duplication trap when a shortened name didn't resolve exactly. (Strong
            # candidates already stopped at the duplicate gate above; these are the weak
            # ones, surfaced as a warning.)
            near_dupes = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
            # M2: don't let a 40-char-truncated slug silently collide with a different
            # entity already in the index — disambiguate to slug-2, slug-3, ...
            slug = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
            path = idx_mod.derive_path(kind, slug, date=date, domain=domain)
            note_aliases = sorted((set(index_aliases) | set(idx_mod.derive_aliases(title))) - {slug})
            note_tags = sorted({kind, *tags})  # baseline `- <kind>` tag; --tags enriches
            note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title,
                               body_text, sources, note_aliases, tags=note_tags)
            echo.cmd_put(path, echo.temp_file(note.encode()))
            action = "created"

        # H3: the entity-index write is the race the review flagged — a bare load->save lets
        # two concurrent captures clobber each other's entry. Route it through a lock-guarded,
        # fresh-re-read transaction. Returns the fresh index (used by auto-link below).
        import echo_concurrency
        index = echo_concurrency.atomic_index_update(
            lambda idx: idx_mod.upsert(idx, slug, path, kind, index_title, index_aliases))

        # auto-link: any other known entity CONFIDENTLY named in the body (M2: is_confident_link
        # rejects short/common tokens so a 3-char alias or a word like "API" can't link everywhere).
        linked = 0
        for s2, e2 in index.get("entities", {}).items():
            if e2.get("path") in (None, path):
                continue
            names = [n for n in [e2.get("title", ""), s2, *e2.get("aliases", [])] if n]
            if any(echo_quality.is_confident_link(n, body_text) for n in names):
                ca, cb = links.link_bidirectional(path, e2["path"])
                linked += int(ca or cb)

        # Keep the BM25 recall index current for this note (best-effort; lock-guarded inside
        # update_note, so it can't clobber a concurrent writer either).
        try:
            import echo_recall
            echo_recall.update_note(path, links.get_text(path) or "")
        except Exception as exc:  # never let recall-index upkeep fail a capture
            print(f"echo_ops: recall-index update skipped ({exc})", file=sys.stderr)

        if not no_log:
            ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]")

    if as_json:
        done(action, path, links=linked, near=near_dupes)
    else:
        print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else ""))
        if near_dupes:
            kind_word = "entity" if len(near_dupes) == 1 else "entities"
            print(f"WARNING: similar existing {kind_word} ({', '.join(near_dupes)}) — if this is "
                  f"the same thing, merge or `echo.py link` instead of keeping a duplicate.",
                  file=real_stdout)
    return 0