echo/mcp-server/app.py

#!/usr/bin/env python3
"""echo-mcp — the containerized MCP server over the ECHO ops layer.  [2.2]

Streamable-HTTP (stateless JSON) MCP server exposing the plugin's `*_op` cores
(Phase 0, shipped 2.1.1) as typed tools. Runs next to the Obsidian REST API on
the same box, so every vault round-trip behind a tool call is LAN-local.

Spec: docs/MCP-SERVER-SPEC.md. Highlights implemented here:
  * bearer-token auth on everything except /health (constant-time compare);
  * tool results = the op envelopes, returned as structured JSON;
  * duplicate gate / offline queueing are DATA (never protocol errors), so the
    model deliberately chooses merge_into/force instead of blind-retrying;
  * tool profiles: ECHO_MCP_TOOLS=core exposes only the six daily drivers
    (schema tokens cost context on every surface that lists them);
  * result budgets: echo_recall(budget_chars), echo_get_note(section/max_chars);
  * per-write serialization (one process mediates all MCP writes, so a plain
    lock removes the advisory-lock race window for server-mediated writes);
  * startup validation: fail fast (crash-loop visibly) on missing env.

Deliberately NOT here in v1 (spec §7.2 backlog): entity-index TTL cache — the
atomic_index_update correctness fix depends on a FRESH re-read under the lock,
so caching idx_mod.load would reintroduce the clobber race it closed. LAN
adjacency + the warm keep-alive pool make the read ~ms anyway.

Env: ECHO_BASE, ECHO_KEY, ECHO_MCP_TOKEN (required) · ECHO_OWNER ·
     ECHO_MCP_PORT=8765 · ECHO_MCP_TOOLS=full|core · ECHO_STATE_DIR=/data
"""
from __future__ import annotations

import contextlib
import hmac
import json
import os
import re
import subprocess
import sys
import threading
from pathlib import Path

# --- locate the plugin scripts (image layout first, repo layout for local dev) ---
_HERE = Path(__file__).resolve().parent
for _cand in (_HERE / "scripts",
              _HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"):
    if _cand.is_dir():
        SCRIPTS = _cand
        break
else:  # pragma: no cover
    sys.exit("echo-mcp: cannot locate the ECHO scripts directory")
sys.path.insert(0, str(SCRIPTS))

# --- startup validation: a misdeployed container must crash-loop visibly ---------
_missing = [k for k in ("ECHO_BASE", "ECHO_KEY", "ECHO_MCP_TOKEN") if not os.environ.get(k)]
if _missing:
    sys.exit(f"echo-mcp: missing required env: {', '.join(_missing)} "
             "— check the PORT template / secret refs")

# Modules aliased *_mod: several tool functions below share a module's name
# (echo_recall, echo_reflect, ...) and would shadow it at module scope otherwise.
import echo                        # noqa: E402
import echo_doctor as doctor_mod   # noqa: E402
import echo_index as idx_mod       # noqa: E402
import echo_ops as ops_mod         # noqa: E402
import echo_queue as queue_mod     # noqa: E402
import echo_recall as recall_mod   # noqa: E402
import echo_reflect as reflect_mod # noqa: E402
import echo_session as session_mod # noqa: E402
import echo_triage as triage_mod   # noqa: E402

from mcp.server.fastmcp import FastMCP  # noqa: E402
from starlette.requests import Request  # noqa: E402
from starlette.responses import JSONResponse, Response  # noqa: E402
from starlette.middleware.base import BaseHTTPMiddleware  # noqa: E402

PORT = int(os.environ.get("ECHO_MCP_PORT", "8765"))
TOKEN = os.environ["ECHO_MCP_TOKEN"]
PROFILE = os.environ.get("ECHO_MCP_TOOLS", "full").strip().lower()
KINDS = sorted(idx_mod.KIND_FOLDER)

mcp = FastMCP(
    "echo-mcp",
    instructions=(
        "Persistent memory over the operator's ECHO Obsidian vault. Use echo_load at "
        "the start of substantive sessions, echo_recall/echo_resolve to read, "
        "echo_capture as the default write, and echo_log_session to end a session. "
        "A duplicate-gate result is a decision point, not an error — merge_into or "
        "(confirmed-distinct) force. Write about the operator in third person."),
    host="0.0.0.0",
    port=PORT,
    stateless_http=True,
    json_response=True,
)

WRITE_LOCK = threading.Lock()   # all MCP-path writes serialize through this process


def _guard(write: bool, fn, *args, **kw) -> dict:
    """Run an op core with stdout redirected to stderr (belt-and-braces stream
    purity) and EchoError mapped to an actionable error envelope, not a protocol
    error. Write ops serialize on WRITE_LOCK."""
    lock = WRITE_LOCK if write else contextlib.nullcontext()
    try:
        with lock, contextlib.redirect_stdout(sys.stderr):
            return fn(*args, **kw)
    except RuntimeError as exc:  # EchoError (either module instance) subclasses it
        code = getattr(exc, "code", 1)
        err = {"ok": False, "error": str(exc), "code": code}
        if code == 78:
            err["hint"] = ("server deployment is missing vault credentials — operator: "
                           "check the PORT template env (SECRET: refs)")
        elif code == 44:
            err["code_name"] = "not-found"
        elif "unreachable" in str(exc):
            err["hint"] = ("vault unreachable (Obsidian/REST plugin likely down) — "
                           "proceed without memory; writes queue durably")
        return err


_SAFE_PATH = re.compile(r"^[^/][^\0]*$")


def _check_path(path: str) -> str | None:
    if not path or path.startswith("/") or ".." in path.split("/") or not _SAFE_PATH.match(path):
        return "invalid path: must be vault-relative, no leading '/' and no '..'"
    return None


def _frontmatter(text: str) -> dict:
    out: dict = {}
    if text.startswith("---"):
        end = text.find("\n---", 3)
        for ln in text[3:end if end != -1 else len(text)].splitlines():
            m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", ln)
            if m:
                out[m.group(1)] = m.group(2).strip().strip('"').strip("'")
    return out


# ------------------------------------------------------------------ tools -------
CORE_TOOLS = {"echo_load", "echo_recall", "echo_capture", "echo_triage_inbox",
              "echo_log_session", "echo_health"}


def tool(fn):
    """Register `fn` as an MCP tool unless the core profile excludes it."""
    if PROFILE == "core" and fn.__name__ not in CORE_TOOLS:
        return fn
    return mcp.tool()(fn)


@tool
def echo_load(brief: bool = True) -> dict:
    """Cold-start memory orientation — call at the start of a substantive session.
    brief=true (default) returns a token-budgeted digest plus structured sections;
    brief=false returns the six raw sections. Also flushes writes queued offline."""
    return _guard(True, echo.load_op, brief)


@tool
def echo_recall(query: str, limit: int = 6, budget_chars: int = 4000,
                include_linked: bool = True) -> dict:
    """Search memory: ranked matches for a topic/person/project PLUS their linked
    neighbourhood. Excerpts are packed into budget_chars by score — call
    echo_get_note for any hit's full content. Freshness/status-aware ranking."""
    limit = max(1, min(int(limit), 20))
    budget = max(500, min(int(budget_chars), 20000))
    env = _guard(False, recall_mod.recall_op, query, limit)
    if not env.get("ok"):
        return env
    if not include_linked:
        env["linked"] = []
    used = 0
    for hit in (env.get("primary") or []) + (env.get("linked") or []):
        ex = hit.get("excerpt") or ""
        room = max(0, budget - used)
        if len(ex) > room:
            hit["excerpt"] = ex[:room] + ("… (truncated — echo_get_note for full)" if room else "")
            hit["truncated"] = True
        used += len(hit.get("excerpt") or "")
    return env


@tool
def echo_resolve(mention: str) -> dict:
    """Resolve a name/mention to its canonical vault note (alias-aware), or get
    did-you-mean candidates. Call before creating any note by hand; echo_capture
    does this automatically."""
    return _guard(False, ops_mod.resolve_op, mention)


@tool
def echo_get_note(path: str, section: str = "", max_chars: int = 8000) -> dict:
    """Read one vault note (vault-relative path). section='Status' returns only that
    ## section. Content over max_chars is truncated with a marker."""
    bad = _check_path(path)
    if bad:
        return {"ok": False, "error": bad}

    def _read() -> dict:
        status, body = echo.request("GET", echo.vault_url(path))
        if status == 404:
            return {"ok": False, "code_name": "not-found", "path": path,
                    "error": f"{path}: not found"}
        echo.check(status, body, f"get {path}")
        text = body.decode(errors="replace")
        fm = _frontmatter(text)
        if section:
            text = echo.extract_heading(text, section)
            if not text:
                return {"ok": False, "path": path,
                        "error": f"no '## {section}' section in {path}"}
        truncated = len(text) > max_chars
        return {"ok": True, "path": path, "frontmatter": fm,
                "content": text[:max_chars] + ("\n… (truncated)" if truncated else ""),
                "truncated": truncated}
    return _guard(False, _read)


@tool
def echo_get_scope() -> dict:
    """The operator's active scope + freshness. If sessions_since >= 3, treat the
    recorded scope as suspect and confirm with the operator before working."""
    return _guard(False, echo.scope_show_op)


@tool
def echo_set_scope(scope: str) -> dict:
    """Switch the active scope atomically (archives the prior scope to Scope History
    and stamps freshness). Use when the session's work diverges from the recorded
    scope."""
    return _guard(True, echo.scope_set_op, scope)


@tool
def echo_health(deep: bool = False) -> dict:
    """ECHO readiness: config, vault reachability, auth, bootstrap/schema, and the
    offline-queue depth. deep=true also runs the full vault-invariant linter."""
    env = _guard(False, doctor_mod.run_op)
    try:
        env["outbox_depth"] = len(queue_mod.pending())
        env["needs_attention"] = len(queue_mod.needs_attention())
    except Exception:  # noqa: BLE001
        pass
    if deep and env.get("ok"):
        r = subprocess.run([sys.executable, str(SCRIPTS / "vault_lint.py")],
                           capture_output=True, text=True,
                           env=dict(os.environ), timeout=120)
        env["lint_exit"] = r.returncode
        env["lint_report"] = (r.stdout or r.stderr)[-6000:]
    return env


@tool
def echo_capture(title: str, kind: str = "", body: str = "", tags: list[str] = [],
                 aliases: list[str] = [], sources: list[str] = [], status: str = "",
                 date: str = "", domain: str = "business", merge_into: str = "",
                 force: bool = False, dry_run: bool = False) -> dict:
    """The default memory write: routes by kind (person, company, concept, reference,
    meeting, project, area, semantic, episodic, working, skill, decision), stamps
    complete frontmatter, indexes, auto-links, and writes the Agent-Log line — one
    call. Omit kind for an inbox capture. If the result is action='duplicate-gate',
    do NOT retry blindly: call again with merge_into=<candidate slug> to update the
    existing entity, or force=true only after confirming they are genuinely distinct.
    Offline, the whole capture queues durably (queued=true)."""
    if kind and kind not in KINDS:
        return {"ok": False, "error": f"unknown kind '{kind}' — one of: {', '.join(KINDS)}"}
    return _guard(True, ops_mod.capture_op, kind or None, title,
                  body_text=body or "", status_v=status, aliases=aliases,
                  sources=sources, tags=tags, date=date or None, domain=domain,
                  inbox=not kind, dry_run=dry_run, force=force,
                  merge_into=merge_into or None)


@tool
def echo_link(a: str, b: str) -> dict:
    """Add reciprocal '## Related' links between two notes. Accepts vault paths or
    resolvable entity names (aliases work)."""
    def _to_path(x: str) -> str | None:
        if "/" in x:
            return x if x.endswith(".md") else x + ".md"
        nmap = idx_mod.name_map(idx_mod.load())
        return nmap.get(idx_mod.slugify(x))
    def _do() -> dict:
        pa, pb = _to_path(a), _to_path(b)
        if not pa or not pb:
            missing = a if not pa else b
            return {"ok": False, "error": f"'{missing}' resolves to no note — "
                                          "pass a vault path or a known entity name"}
        return ops_mod.link_op(pa, pb)
    return _guard(True, _do)


@tool
def echo_append_note(path: str, line: str) -> dict:
    """Append one line to a note, idempotently (skipped if the exact line already
    exists). For inbox lines, Agent-Log entries, Observations bullets."""
    bad = _check_path(path)
    if bad:
        return {"ok": False, "error": bad}
    def _do() -> dict:
        rc = echo.cmd_append(path, line)
        return {"ok": rc == 0, "action": "append", "path": path}
    return _guard(True, _do)


@tool
def echo_patch_note(path: str, operation: str, target_type: str, target: str,
                    content: str) -> dict:
    """Targeted edit: append/prepend/replace under a heading, frontmatter field, or
    block. HEADING TARGETS must be the full '::'-delimited path from the top-level
    heading (e.g. 'Operator Preferences::Fact / Pattern') — on an invalid target the
    error includes the note's actual headings. replace overwrites the section."""
    bad = _check_path(path)
    if bad:
        return {"ok": False, "error": bad}
    def _do() -> dict:
        rc = echo.cmd_patch(path, operation, target_type, target,
                            echo.temp_file(content.encode("utf-8")))
        return {"ok": rc == 0, "action": f"patch:{operation}", "path": path,
                "target": target}
    env = _guard(True, _do)
    if not env.get("ok") and "HTTP 400" in str(env.get("error", "")) and target_type == "heading":
        st, body = echo.request("GET", echo.vault_url(path),
                                headers={"Accept": "application/vnd.olrapi.document-map+json"})
        if st == 200:
            try:
                headings = [h.get("heading") or h for h in
                            json.loads(body).get("headings", [])][:40]
                env["available_headings"] = headings
                env["hint"] = "use one of available_headings verbatim as the Target"
            except Exception:  # noqa: BLE001
                pass
    return env


@tool
def echo_triage_inbox(proposals: list[dict] = [], apply: bool = False) -> dict:
    """Inbox triage. No proposals => structured listing of captures (line, date, text,
    age_days). With proposals (reflect schema + optional 'line'): apply=false previews
    the routing; apply=true routes via capture and writes the processing-log audit.
    List first, propose, preview, then apply only after the operator confirms."""
    if not proposals:
        return _guard(False, triage_mod.list_op)
    return _guard(True, triage_mod.route_op, proposals, apply)


@tool
def echo_reflect(proposals: list[dict], apply: bool = False) -> dict:
    """Session-reflection proposals: validate -> classify against the entity index ->
    preview (apply=false) -> apply (routes each via capture). Never apply without the
    operator's go-ahead; never invent memories to have something to save."""
    return _guard(True, reflect_mod.apply_op, proposals, apply)


@tool
def echo_log_session(slug: str, log_body: str, agent_log_line: str = "",
                     scope: str = "", reflect: list[dict] = [],
                     apply: bool = False, hhmm: str = "") -> dict:
    """End a substantive session in ONE call: session log -> Agent-Log line -> reflect
    proposals -> optional scope switch -> heartbeat LAST (the commit marker).
    apply=false previews the plan. Pass hhmm (local time, e.g. '1430') so the log
    filename sorts truthfully."""
    bundle = {"slug": slug, "log_body": log_body}
    if agent_log_line:
        bundle["agent_log_line"] = agent_log_line
    if scope:
        bundle["scope"] = scope
    if reflect:
        bundle["reflect"] = reflect
    def _do() -> dict:
        prev = os.environ.get("ECHO_NOW")
        try:
            if hhmm:
                os.environ["ECHO_NOW"] = hhmm
            return session_mod.session_end_op(bundle, apply=apply)
        finally:
            if hhmm:
                if prev is None:
                    os.environ.pop("ECHO_NOW", None)
                else:
                    os.environ["ECHO_NOW"] = prev
    return _guard(True, _do)


# ------------------------------------------------------------- health + auth ----
@mcp.custom_route("/health", methods=["GET"])
async def health(_request: Request) -> JSONResponse:
    """Unauthenticated liveness for Docker HEALTHCHECK + Kuma. No vault data."""
    st, _ = echo.request("GET", echo.vault_url("_agent/echo-vault.md"))
    try:
        outbox = len(queue_mod.pending())
    except Exception:  # noqa: BLE001
        outbox = -1
    return JSONResponse({"ok": True, "vault_reachable": st != 0,
                         "outbox_depth": outbox, "profile": PROFILE})


class BearerAuth(BaseHTTPMiddleware):
    async def dispatch(self, request, call_next):
        if request.url.path == "/health":
            return await call_next(request)
        auth = request.headers.get("authorization", "")
        if not hmac.compare_digest(auth, f"Bearer {TOKEN}"):
            return Response(status_code=401)
        return await call_next(request)


def main() -> None:
    import uvicorn
    app = mcp.streamable_http_app()
    app.add_middleware(BearerAuth)
    print(f"echo-mcp: serving on :{PORT} (profile={PROFILE}, vault={echo.BASE})",
          file=sys.stderr)
    uvicorn.run(app, host="0.0.0.0", port=PORT, log_level="info")


if __name__ == "__main__":
    main()