chorus/eval/run_eval.py

#!/usr/bin/env python3
"""run_eval.py — current-version metrics for chorus-memory (v1.5+).

Replaces the historical 0.6-vs-0.7 A/B harness (see git history for that version).
Measures the four things the plugin now claims, against the deterministic mock
REST API (mock_olrapi.py) — no credentials, no live vault:

  1. RETRIEVAL   — hybrid recall (BM25 over entities + sessions/journal, fused with
                   freshness/status priors) vs a keyword-only baseline modeling the
                   pre-1.5 recall (per-word substring ranking over ENTITY notes only).
                   Metrics: precision@5, recall@5, MRR, session/journal queries
                   answered, freshness top-1 correctness.
  2. DEDUP       — duplicate notes created replaying a capture stream with the
                   pre-write gate ON (default) vs OFF (CHORUS_DUP_GATE=99, i.e. the
                   pre-1.5 warn-only behavior). Also counts legitimate captures
                   wrongly blocked (must be 0).
  3. WRITE SAFETY— silent write failures (claimed success but ground truth wrong)
                   across fault injection: transient 503, phantom PUT, PATCH to a
                   missing heading, replayed append. Must be 0; failures must be LOUD.
  4. DURABILITY  — writes issued while the vault is unreachable must queue, replay
                   exactly once when it returns, and never duplicate on re-flush.

Ground truth is always read back independently via the mock's __debug__ endpoint.
Results print as a table and land in results/latest.json.

Usage:  python3 run_eval.py [--port 8797]
"""
from __future__ import annotations

import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
from pathlib import Path

HERE = Path(__file__).resolve().parent
SCRIPTS = HERE.parent / "chorus-memory.plugin.src" / "skills" / "chorus-memory" / "scripts"
CHORUS = SCRIPTS / "chorus.py"
SWEEP = SCRIPTS / "sweep.py"
KEY = "test-key-not-a-real-secret"
TODAY = "2026-07-03"


def http(method, url, body=None, headers=None):
    data = body.encode() if isinstance(body, str) else body
    req = urllib.request.Request(url, data=data, method=method,
                                 headers={"Authorization": f"Bearer {KEY}", **(headers or {})})
    try:
        with urllib.request.urlopen(req, timeout=10) as r:
            return r.status, r.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        return e.code, e.read().decode("utf-8", "replace")
    except Exception as e:  # noqa: BLE001
        return 0, str(e)


class Harness:
    def __init__(self, base, state_dir):
        self.base = base
        self.state_dir = state_dir

    def reset(self):
        http("POST", f"{self.base}/__debug__reset")

    def seed(self, path, content):
        http("PUT", f"{self.base}/vault/{path}", content)

    def ground(self, path):
        _, body = http("GET", f"{self.base}/__debug__?path={path}")
        return None if body == "<<MISSING>>" else body

    def run(self, script, *args, env_extra=None, base=None):
        env = dict(os.environ, CHORUS_BASE=base or self.base, CHORUS_KEY=KEY,
                   CHORUS_VERIFY="1", CHORUS_TODAY=TODAY, CHORUS_STATE_DIR=self.state_dir,
                   **(env_extra or {}))
        return subprocess.run([sys.executable, str(script), *args],
                              capture_output=True, text=True, env=env)


def note(fm_type, title, body, status="active", created=TODAY, updated=TODAY, tags=None):
    tags_s = "[" + ", ".join(tags or [fm_type]) + "]"
    return (f"---\ntype: {fm_type}\nstatus: {status}\ncreated: {created}\n"
            f"updated: {updated}\ntags: {tags_s}\nagent_written: true\nsource_notes: []\n---\n\n"
            f"# {title}\n\n{body}\n\n## Related\n")


# ---------------------------------------------------------------- 1. RETRIEVAL
ENTITY_CORPUS = {
    "resources/people/maria-lopez.md": note(
        "person", "Maria Lopez", "Runs procurement at Vantage Systems. Prefers email over calls."),
    "resources/companies/vantage-systems.md": note(
        "company", "Vantage Systems", "Enterprise procurement platform vendor; the contract renews in Q3."),
    "resources/concepts/bm25-ranking.md": note(
        "concept", "BM25 Ranking", "Lexical retrieval scoring with term frequency saturation and document length normalization."),
    "resources/concepts/graph-expansion.md": note(
        "concept", "Graph Expansion", "Expanding a hit into its linked neighbourhood along Related links and source notes."),
    "projects/active/website-redesign.md": note(
        "project", "Website Redesign",
        "Rebuild the marketing site. Navigation cleanup and new hero copy.",
        status="active", created="2026-06-25", updated="2026-07-01"),
    "projects/archived/old-website.md": note(
        "project", "Old Website",
        "Legacy marketing site. Navigation used nested dropdown menus. Navigation was the top complaint.",
        status="archived", created="2024-03-01", updated="2024-08-10"),
    "_agent/memory/semantic/tooling-preferences.md": note(
        "semantic-memory", "Tooling Preferences",
        "The operator prefers uv over pip for Python tooling, and ruff for linting."),
}
TIME_SERIES_CORPUS = {
    "_agent/sessions/2026-06-20-1000-vantage-contract-review.md": note(
        "session-log", "Vantage contract review",
        "Reviewed the Vantage contract. Decided to renegotiate the SLA penalty clause before renewal.",
        status="complete", created="2026-06-20", updated="2026-06-20", tags=["session-log"]),
    "journal/daily/2026-06-20.md": (
        "---\ntype: daily-note\ncreated: 2026-06-20\nupdated: 2026-06-20\ntags: [daily]\n---\n\n"
        "# 2026-06-20\n\n## Agent Log\n- 2026-06-20: renegotiation kickoff email sent to Vantage\n"),
}

# (query, gold paths, only answerable from sessions/journal)
QUERIES = [
    ("uv or pip for python tooling", {"_agent/memory/semantic/tooling-preferences.md"}, False),
    ("procurement contact", {"resources/people/maria-lopez.md",
                             "resources/companies/vantage-systems.md"}, False),
    ("SLA penalty clause decision",
     {"_agent/sessions/2026-06-20-1000-vantage-contract-review.md"}, True),
    ("marketing site navigation", {"projects/active/website-redesign.md",
                                   "projects/archived/old-website.md"}, False),
    ("term frequency saturation", {"resources/concepts/bm25-ranking.md"}, False),
    ("contract renewal Q3", {"resources/companies/vantage-systems.md"}, False),
    ("renegotiation kickoff", {"journal/daily/2026-06-20.md"}, True),
    ("graph neighbourhood expansion", {"resources/concepts/graph-expansion.md"}, False),
]
FRESHNESS_QUERY = "marketing site navigation"
FRESHNESS_TOP1 = "projects/active/website-redesign.md"


def keyword_baseline(query, corpus, k=5):
    """Models pre-1.5 keyword recall: per-word substring counting over ENTITY note
    bodies only (sessions/journal were not in the corpus before 1.5)."""
    words = [w for w in re.findall(r"[a-z0-9]+", query.lower()) if len(w) > 2]
    scored = []
    for path, text in corpus.items():
        body = text.lower()
        s = sum(body.count(w) for w in words)
        if s > 0:
            scored.append((s, path))
    scored.sort(key=lambda x: (-x[0], x[1]))
    return [p for _, p in scored[:k]]


def metrics_for(ranked_by_query):
    p_sum = r_sum = mrr_sum = 0.0
    ts_answered = ts_total = 0
    for (q, gold, ts_only), ranked in ranked_by_query:
        top = ranked[:5]
        hits = [p for p in top if p in gold]
        p_sum += len(hits) / 5
        r_sum += len(hits) / len(gold)
        rr = 0.0
        for i, p in enumerate(top):
            if p in gold:
                rr = 1.0 / (i + 1)
                break
        mrr_sum += rr
        if ts_only:
            ts_total += 1
            ts_answered += int(bool(hits))
    n = len(ranked_by_query)
    return {"precision_at_5": round(p_sum / n, 3), "recall_at_5": round(r_sum / n, 3),
            "mrr": round(mrr_sum / n, 3),
            "session_journal_queries_answered": f"{ts_answered}/{ts_total}"}


def eval_retrieval(h):
    h.reset()
    h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
    for path, text in {**ENTITY_CORPUS, **TIME_SERIES_CORPUS}.items():
        h.seed(path, text)
    r = h.run(SWEEP, "--apply")
    assert r.returncode == 0, f"sweep failed: {r.stdout}{r.stderr}"

    current_ranked, baseline_ranked = [], []
    fresh_top1_current = fresh_top1_baseline = None
    for q, gold, ts_only in QUERIES:
        r = h.run(CHORUS, "recall", q, "--json", "--limit", "5")
        try:
            primary = [hit["path"] for hit in json.loads(r.stdout).get("primary", [])]
        except Exception:  # noqa: BLE001
            primary = []
        current_ranked.append(((q, gold, ts_only), primary))
        base = keyword_baseline(q, ENTITY_CORPUS)
        baseline_ranked.append(((q, gold, ts_only), base))
        if q == FRESHNESS_QUERY:
            fresh_top1_current = primary[0] if primary else None
            fresh_top1_baseline = base[0] if base else None

    cur = metrics_for(current_ranked)
    cur["freshness_top1_correct"] = fresh_top1_current == FRESHNESS_TOP1
    bas = metrics_for(baseline_ranked)
    bas["freshness_top1_correct"] = fresh_top1_baseline == FRESHNESS_TOP1
    return {"current (hybrid+priors, full corpus)": cur,
            "baseline (keyword, entities only)": bas}


# ------------------------------------------------------------------- 2. DEDUP
# Attempts referring to an EXISTING entity under another name; a create == duplicate.
DUP_ATTEMPTS = [
    ("Robert Smith", "person", "resources/people/robert-smith.md"),
    ("Chorus Memory", "project", "projects/active/chorus-memory.md"),
    ("Acme Consulting", "company", "resources/companies/acme-consulting.md"),
]
# Legitimate NEW entities; a block == false positive (must create in both modes).
LEGIT_ATTEMPTS = [
    ("Chorus Review Meeting", "meeting",
     f"resources/meetings/{TODAY}-chorus-review-meeting.md"),   # cross-kind lookalike
    ("Zephyr", "concept", "resources/concepts/zephyr.md"),    # unrelated
]


def _dedup_pass(h, gate_env):
    h.reset()
    h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
    for title, kind in [("Bob Smith", "person"), ("Chorus", "project"), ("Acme", "company")]:
        h.run(CHORUS, "capture", title, "--kind", kind, env_extra=gate_env)
    dupes = blocked_legit = 0
    for title, kind, would_be in DUP_ATTEMPTS:
        h.run(CHORUS, "capture", title, "--kind", kind, env_extra=gate_env)
        dupes += int(h.ground(would_be) is not None)
    for title, kind, path in LEGIT_ATTEMPTS:
        h.run(CHORUS, "capture", title, "--kind", kind, env_extra=gate_env)
        blocked_legit += int(h.ground(path) is None)
    return {"duplicate_notes_created": dupes, "legitimate_captures_blocked": blocked_legit,
            "referring_attempts": len(DUP_ATTEMPTS), "legit_attempts": len(LEGIT_ATTEMPTS)}


def eval_dedup(h):
    return {"gate ON (v1.5.1 default)": _dedup_pass(h, {}),
            "gate OFF (pre-1.5, warn-only)": _dedup_pass(h, {"CHORUS_DUP_GATE": "99"})}


# ------------------------------------------------------------- 3. WRITE SAFETY
def eval_write_safety(h):
    h.reset()
    h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
    silent = loud = 0
    ops = 0

    def bodyfile(text):
        f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
        f.write(text)
        f.close()
        return f.name

    # transient 503 on first write ("flaky" marker) -> client must retry and land it
    ops += 1
    r = h.run(CHORUS, "put", "resources/concepts/flaky-note.md", bodyfile("# Flaky\nok\n"))
    landed = h.ground("resources/concepts/flaky-note.md") is not None
    if r.returncode == 0 and not landed:
        silent += 1
    if r.returncode != 0:
        loud += 1

    # phantom PUT ("phantom" marker: 200 but not persisted) -> verify must fail LOUD
    ops += 1
    r = h.run(CHORUS, "put", "resources/concepts/phantom-note.md", bodyfile("# Phantom\n"))
    landed = h.ground("resources/concepts/phantom-note.md") is not None
    if r.returncode == 0 and not landed:
        silent += 1
    if r.returncode != 0 and not landed:
        loud += 1

    # PATCH to a missing heading -> 400 must be surfaced, nothing written
    ops += 1
    h.seed("resources/concepts/target.md", "# Target\n\n## Real\nx\n")
    r = h.run(CHORUS, "patch", "resources/concepts/target.md", "append", "heading",
              "Target::Nope", bodyfile("lost?"))
    if r.returncode == 0 and "lost?" not in (h.ground("resources/concepts/target.md") or ""):
        silent += 1
    if r.returncode != 0:
        loud += 1

    # replayed append -> whole-line idempotency, no duplicate
    ops += 1
    h.run(CHORUS, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
    h.run(CHORUS, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
    n = (h.ground("inbox/captures/inbox.md") or "").count("replay me")
    if n != 1:
        silent += 1

    return {"fault_scenarios": ops, "silent_write_failures": silent,
            "failures_surfaced_loud": loud}


# --------------------------------------------------------------- 4. DURABILITY
def eval_durability(h, dead_base):
    h.reset()
    h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
    queued_ok = 0
    for i in range(3):
        r = h.run(CHORUS, "append", "inbox/captures/inbox.md",
                  f"- {TODAY}: offline capture {i}", base=dead_base)
        queued_ok += int(r.returncode == 0 and "queued" in r.stdout)
    # vault returns: flush replays; a second flush must not duplicate
    h.run(CHORUS, "flush")
    h.run(CHORUS, "flush")
    inbox = h.ground("inbox/captures/inbox.md") or ""
    landed = sum(1 for i in range(3) if f"offline capture {i}" in inbox)
    dupes = sum(inbox.count(f"offline capture {i}") - 1
                for i in range(3) if f"offline capture {i}" in inbox)
    return {"writes_during_outage": 3, "reported_queued": queued_ok,
            "landed_after_flush": landed, "lost": 3 - landed, "duplicated_on_reflush": dupes}


# ------------------------------------------------------------------------ main
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--port", type=int, default=8797)
    a = ap.parse_args()
    base = f"http://127.0.0.1:{a.port}"
    srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
    state = tempfile.mkdtemp(prefix="chorus-eval-state-")
    try:
        for _ in range(50):
            try:
                urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1)
                break
            except Exception:  # noqa: BLE001
                time.sleep(0.1)
        h = Harness(base, state)
        results = {
            "version": "1.5.1", "date": TODAY,
            "retrieval": eval_retrieval(h),
            "dedup": eval_dedup(h),
            "write_safety": eval_write_safety(h),
            "durability": eval_durability(h, dead_base=f"http://127.0.0.1:{a.port + 1}"),
        }
        out = HERE / "results" / "latest.json"
        out.parent.mkdir(exist_ok=True)
        out.write_text(json.dumps(results, indent=2), encoding="utf-8")

        print(f"\n== chorus-memory eval — v{results['version']} ({results['date']}) ==\n")
        for section, data in results.items():
            if section in ("version", "date"):
                continue
            print(f"## {section}")
            for k, v in data.items():
                if isinstance(v, dict):
                    print(f"  {k}:")
                    for k2, v2 in v.items():
                        print(f"    {k2:38} {v2}")
                else:
                    print(f"  {k:40} {v}")
            print()
        print(f"results -> {out}")
        return 0
    finally:
        srv.terminate()


if __name__ == "__main__":
    raise SystemExit(main())