echo/eval/test_mcp_server.py

#!/usr/bin/env python3
"""test_mcp_server.py — echo-mcp (2.2) end-to-end against the mock vault.

Spawns mcp-server/app.py + mock_olrapi and drives the real streamable-HTTP MCP
protocol: health (open), auth (401), initialize, tools/list, and the memory-day
tools/call flow (capture -> duplicate gate as DATA -> merge_into -> recall ->
get_note -> log_session with heartbeat commit).

Requires the `mcp` package in the interpreter that runs the SERVER. Set
ECHO_MCP_PYTHON to a venv python that has it; otherwise the current interpreter
is tried and the suite SKIPS (exit 0) when the SDK is absent — the other suites
don't depend on it.

Run:  python test_mcp_server.py [--port 8862] [--mcp-port 8767]
"""
from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent
APP = HERE.parent / "mcp-server" / "app.py"
KEY = "test-key-not-a-real-secret"
TOKEN = "local-test-token"

failures = []


def check(name, cond, detail=""):
    print(f"{'ok  ' if cond else 'FAIL'} {name}" + (f"  -- {detail}" if not cond else ""))
    if not cond:
        failures.append(name)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--port", type=int, default=8862)
    ap.add_argument("--mcp-port", type=int, default=8767)
    a = ap.parse_args()
    mock_base = f"http://127.0.0.1:{a.port}"
    mcp_url = f"http://127.0.0.1:{a.mcp_port}/mcp"

    server_py = os.environ.get("ECHO_MCP_PYTHON") or sys.executable
    probe = subprocess.run([server_py, "-c", "import mcp"], capture_output=True)
    if probe.returncode != 0:
        print("SKIP: `mcp` SDK not installed for the server interpreter "
              "(set ECHO_MCP_PYTHON to a venv python that has it)")
        return 0

    def http(method, url, body=None, headers=None, timeout=30):
        data = body.encode() if isinstance(body, str) else body
        req = urllib.request.Request(url, data=data, method=method, headers=headers or {})
        try:
            with urllib.request.urlopen(req, timeout=timeout) as r:
                return r.status, r.read().decode("utf-8", "replace")
        except urllib.error.HTTPError as e:
            return e.code, e.read().decode("utf-8", "replace")
        except Exception as e:  # noqa: BLE001
            return 0, str(e)

    _id = [0]

    def rpc(method, params):
        _id[0] += 1
        st, body = http("POST", mcp_url, json.dumps(
            {"jsonrpc": "2.0", "id": _id[0], "method": method, "params": params}),
            headers={"Content-Type": "application/json",
                     "Accept": "application/json, text/event-stream",
                     "Authorization": f"Bearer {TOKEN}"})
        return st, (json.loads(body) if body.strip().startswith("{") else {})

    def call(name, args):
        st, d = rpc("tools/call", {"name": name, "arguments": args})
        res = d.get("result") or {}
        payload = {}
        if res.get("content"):
            try:
                payload = json.loads(res["content"][0]["text"])
            except Exception:  # noqa: BLE001
                payload = {}
        return res.get("isError", False), payload

    mock = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
    srv = None
    try:
        for _ in range(50):
            try:
                urllib.request.urlopen(f"{mock_base}/__debug__reset", data=b"", timeout=1)
                break
            except Exception:
                time.sleep(0.1)
        http("PUT", f"{mock_base}/vault/_agent/echo-vault.md",
             "---\nschema_version: 4\n---\n# marker\n",
             headers={"Authorization": f"Bearer {KEY}"})

        env = dict(os.environ, ECHO_BASE=mock_base, ECHO_KEY=KEY, ECHO_MCP_TOKEN=TOKEN,
                   ECHO_MCP_PORT=str(a.mcp_port), ECHO_STATE_DIR=tempfile.mkdtemp(),
                   ECHO_TODAY="2026-07-28")
        srv = subprocess.Popen([server_py, str(APP)], env=env,
                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
        for _ in range(60):
            st, _b = http("GET", f"http://127.0.0.1:{a.mcp_port}/health", timeout=2)
            if st == 200:
                break
            time.sleep(0.5)

        st, body = http("GET", f"http://127.0.0.1:{a.mcp_port}/health")
        try:
            hb = json.loads(body)
        except Exception:  # noqa: BLE001
            hb = {}
        check("health is open and green", st == 200 and hb.get("ok") is True
              and hb.get("vault_reachable") is True, body[:200])
        st, _b = http("POST", mcp_url, "{}", headers={"Content-Type": "application/json"})
        check("MCP endpoint requires the bearer token (401)", st == 401, str(st))

        st, d = rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
                                   "clientInfo": {"name": "eval", "version": "0"}})
        check("initialize answers with serverInfo",
              d.get("result", {}).get("serverInfo", {}).get("name") == "echo-mcp", json.dumps(d)[:200])
        st, d = rpc("tools/list", {})
        tools = [t["name"] for t in d.get("result", {}).get("tools", [])]
        check("tools/list exposes the full profile (14 tools)", len(tools) == 14, str(tools))

        e, b = call("echo_capture", {"title": "Vera Lumen", "kind": "person",
                                     "body": "CTO at Fluxcorp, met at the summit."})
        check("capture create over MCP", not e and b.get("action") == "created"
              and b.get("path") == "resources/people/vera-lumen.md", json.dumps(b))
        e, b = call("echo_capture", {"title": "Vera Lumen Jr", "kind": "person"})
        check("duplicate gate is data, not isError", not e
              and b.get("action") == "duplicate-gate" and b.get("candidates"), json.dumps(b))
        e, b = call("echo_capture", {"title": "Vera Lumen Jr", "kind": "person",
                                     "merge_into": "vera-lumen", "body": "Follow-up."})
        check("merge_into resolves the gate as an update", not e and b.get("action") == "updated",
              json.dumps(b))
        e, b = call("echo_recall", {"query": "fluxcorp", "budget_chars": 900})
        check("recall over MCP finds by body term", not e and any(
            h["path"] == "resources/people/vera-lumen.md" for h in b.get("primary", [])),
            json.dumps(b)[:300])
        e, b = call("echo_get_note", {"path": "resources/people/vera-lumen.md"})
        check("get_note returns frontmatter + content", not e
              and b.get("frontmatter", {}).get("type") == "person", json.dumps(b)[:200])
        e, b = call("echo_get_note", {"path": "../etc/passwd"})
        check("get_note rejects path traversal", not b.get("ok"))
        e, b = call("echo_log_session", {"slug": "mcp-e2e", "hhmm": "2345", "apply": True,
                                         "log_body": "---\ntype: session-log\n---\n# S\n\n## Goal\ne2e\n"})
        check("log_session commits with the heartbeat", not e
              and b.get("steps", {}).get("heartbeat") == "ok", json.dumps(b))
        e, b = call("echo_capture", {"title": "Bad Kind", "kind": "wizard"})
        check("unknown kind rejected with the valid list", not b.get("ok")
              and "wizard" in b.get("error", ""), json.dumps(b))

        print(f"\n{len(failures)} failure(s)" if failures else "\nall mcp-server tests passed")
        return 1 if failures else 0
    finally:
        if srv:
            srv.terminate()
        mock.terminate()


if __name__ == "__main__":
    raise SystemExit(main())