forked from jason/echo
Jason Stedwell
993abdc846
Knocks two items off ROADMAP-2.0 (§4 eval refresh, §5 roadmap retirement): - run_eval.py: the stale 0.6-vs-0.7 A/B is replaced (old version in git history) with a four-part current-version eval against the mock — retrieval (hybrid+priors vs keyword/entities-only baseline: R@5 1.00 vs 0.75, MRR 1.00 vs 0.75, session/journal queries 2/2 vs 0/2, freshness top-1 correct vs wrong), dedup (0 dupes gate-on vs 3 gate-off, 0 false blocks), write safety (0 silent failures across 4 fault scenarios), durability (3/3 offline writes queued+landed, 0 lost, 0 re-flush dupes). Results in eval/results/latest.json. - README: metrics table published; repo-layout eval descriptor updated. - eval/README: new harness documented; A/B framing marked historical. - ROADMAP-1.0.md removed (fully shipped; story lives in the README version table + git history); dangling docstring pointer fixed in echo_concurrency.py; ROADMAP-2.0 checkboxes ticked. All suites green; artifact rebuilt (1.5.1, docstring-only source change). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
369 lines
16 KiB
Python
369 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""run_eval.py — current-version metrics for echo-memory (v1.5+).
|
|
|
|
Replaces the historical 0.6-vs-0.7 A/B harness (see git history for that version).
|
|
Measures the four things the plugin now claims, against the deterministic mock
|
|
REST API (mock_olrapi.py) — no credentials, no live vault:
|
|
|
|
1. RETRIEVAL — hybrid recall (BM25 over entities + sessions/journal, fused with
|
|
freshness/status priors) vs a keyword-only baseline modeling the
|
|
pre-1.5 recall (per-word substring ranking over ENTITY notes only).
|
|
Metrics: precision@5, recall@5, MRR, session/journal queries
|
|
answered, freshness top-1 correctness.
|
|
2. DEDUP — duplicate notes created replaying a capture stream with the
|
|
pre-write gate ON (default) vs OFF (ECHO_DUP_GATE=99, i.e. the
|
|
pre-1.5 warn-only behavior). Also counts legitimate captures
|
|
wrongly blocked (must be 0).
|
|
3. WRITE SAFETY— silent write failures (claimed success but ground truth wrong)
|
|
across fault injection: transient 503, phantom PUT, PATCH to a
|
|
missing heading, replayed append. Must be 0; failures must be LOUD.
|
|
4. DURABILITY — writes issued while the vault is unreachable must queue, replay
|
|
exactly once when it returns, and never duplicate on re-flush.
|
|
|
|
Ground truth is always read back independently via the mock's __debug__ endpoint.
|
|
Results print as a table and land in results/latest.json.
|
|
|
|
Usage: python3 run_eval.py [--port 8797]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"
|
|
ECHO = SCRIPTS / "echo.py"
|
|
SWEEP = SCRIPTS / "sweep.py"
|
|
KEY = "test-key-not-a-real-secret"
|
|
TODAY = "2026-07-03"
|
|
|
|
|
|
def http(method, url, body=None, headers=None):
|
|
data = body.encode() if isinstance(body, str) else body
|
|
req = urllib.request.Request(url, data=data, method=method,
|
|
headers={"Authorization": f"Bearer {KEY}", **(headers or {})})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=10) as r:
|
|
return r.status, r.read().decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read().decode("utf-8", "replace")
|
|
except Exception as e: # noqa: BLE001
|
|
return 0, str(e)
|
|
|
|
|
|
class Harness:
|
|
def __init__(self, base, state_dir):
|
|
self.base = base
|
|
self.state_dir = state_dir
|
|
|
|
def reset(self):
|
|
http("POST", f"{self.base}/__debug__reset")
|
|
|
|
def seed(self, path, content):
|
|
http("PUT", f"{self.base}/vault/{path}", content)
|
|
|
|
def ground(self, path):
|
|
_, body = http("GET", f"{self.base}/__debug__?path={path}")
|
|
return None if body == "<<MISSING>>" else body
|
|
|
|
def run(self, script, *args, env_extra=None, base=None):
|
|
env = dict(os.environ, ECHO_BASE=base or self.base, ECHO_KEY=KEY,
|
|
ECHO_VERIFY="1", ECHO_TODAY=TODAY, ECHO_STATE_DIR=self.state_dir,
|
|
**(env_extra or {}))
|
|
return subprocess.run([sys.executable, str(script), *args],
|
|
capture_output=True, text=True, env=env)
|
|
|
|
|
|
def note(fm_type, title, body, status="active", created=TODAY, updated=TODAY, tags=None):
|
|
tags_s = "[" + ", ".join(tags or [fm_type]) + "]"
|
|
return (f"---\ntype: {fm_type}\nstatus: {status}\ncreated: {created}\n"
|
|
f"updated: {updated}\ntags: {tags_s}\nagent_written: true\nsource_notes: []\n---\n\n"
|
|
f"# {title}\n\n{body}\n\n## Related\n")
|
|
|
|
|
|
# ---------------------------------------------------------------- 1. RETRIEVAL
|
|
ENTITY_CORPUS = {
|
|
"resources/people/maria-lopez.md": note(
|
|
"person", "Maria Lopez", "Runs procurement at Vantage Systems. Prefers email over calls."),
|
|
"resources/companies/vantage-systems.md": note(
|
|
"company", "Vantage Systems", "Enterprise procurement platform vendor; the contract renews in Q3."),
|
|
"resources/concepts/bm25-ranking.md": note(
|
|
"concept", "BM25 Ranking", "Lexical retrieval scoring with term frequency saturation and document length normalization."),
|
|
"resources/concepts/graph-expansion.md": note(
|
|
"concept", "Graph Expansion", "Expanding a hit into its linked neighbourhood along Related links and source notes."),
|
|
"projects/active/website-redesign.md": note(
|
|
"project", "Website Redesign",
|
|
"Rebuild the marketing site. Navigation cleanup and new hero copy.",
|
|
status="active", created="2026-06-25", updated="2026-07-01"),
|
|
"projects/archived/old-website.md": note(
|
|
"project", "Old Website",
|
|
"Legacy marketing site. Navigation used nested dropdown menus. Navigation was the top complaint.",
|
|
status="archived", created="2024-03-01", updated="2024-08-10"),
|
|
"_agent/memory/semantic/tooling-preferences.md": note(
|
|
"semantic-memory", "Tooling Preferences",
|
|
"The operator prefers uv over pip for Python tooling, and ruff for linting."),
|
|
}
|
|
TIME_SERIES_CORPUS = {
|
|
"_agent/sessions/2026-06-20-1000-vantage-contract-review.md": note(
|
|
"session-log", "Vantage contract review",
|
|
"Reviewed the Vantage contract. Decided to renegotiate the SLA penalty clause before renewal.",
|
|
status="complete", created="2026-06-20", updated="2026-06-20", tags=["session-log"]),
|
|
"journal/daily/2026-06-20.md": (
|
|
"---\ntype: daily-note\ncreated: 2026-06-20\nupdated: 2026-06-20\ntags: [daily]\n---\n\n"
|
|
"# 2026-06-20\n\n## Agent Log\n- 2026-06-20: renegotiation kickoff email sent to Vantage\n"),
|
|
}
|
|
|
|
# (query, gold paths, only answerable from sessions/journal)
|
|
QUERIES = [
|
|
("uv or pip for python tooling", {"_agent/memory/semantic/tooling-preferences.md"}, False),
|
|
("procurement contact", {"resources/people/maria-lopez.md",
|
|
"resources/companies/vantage-systems.md"}, False),
|
|
("SLA penalty clause decision",
|
|
{"_agent/sessions/2026-06-20-1000-vantage-contract-review.md"}, True),
|
|
("marketing site navigation", {"projects/active/website-redesign.md",
|
|
"projects/archived/old-website.md"}, False),
|
|
("term frequency saturation", {"resources/concepts/bm25-ranking.md"}, False),
|
|
("contract renewal Q3", {"resources/companies/vantage-systems.md"}, False),
|
|
("renegotiation kickoff", {"journal/daily/2026-06-20.md"}, True),
|
|
("graph neighbourhood expansion", {"resources/concepts/graph-expansion.md"}, False),
|
|
]
|
|
FRESHNESS_QUERY = "marketing site navigation"
|
|
FRESHNESS_TOP1 = "projects/active/website-redesign.md"
|
|
|
|
|
|
def keyword_baseline(query, corpus, k=5):
|
|
"""Models pre-1.5 keyword recall: per-word substring counting over ENTITY note
|
|
bodies only (sessions/journal were not in the corpus before 1.5)."""
|
|
words = [w for w in re.findall(r"[a-z0-9]+", query.lower()) if len(w) > 2]
|
|
scored = []
|
|
for path, text in corpus.items():
|
|
body = text.lower()
|
|
s = sum(body.count(w) for w in words)
|
|
if s > 0:
|
|
scored.append((s, path))
|
|
scored.sort(key=lambda x: (-x[0], x[1]))
|
|
return [p for _, p in scored[:k]]
|
|
|
|
|
|
def metrics_for(ranked_by_query):
|
|
p_sum = r_sum = mrr_sum = 0.0
|
|
ts_answered = ts_total = 0
|
|
for (q, gold, ts_only), ranked in ranked_by_query:
|
|
top = ranked[:5]
|
|
hits = [p for p in top if p in gold]
|
|
p_sum += len(hits) / 5
|
|
r_sum += len(hits) / len(gold)
|
|
rr = 0.0
|
|
for i, p in enumerate(top):
|
|
if p in gold:
|
|
rr = 1.0 / (i + 1)
|
|
break
|
|
mrr_sum += rr
|
|
if ts_only:
|
|
ts_total += 1
|
|
ts_answered += int(bool(hits))
|
|
n = len(ranked_by_query)
|
|
return {"precision_at_5": round(p_sum / n, 3), "recall_at_5": round(r_sum / n, 3),
|
|
"mrr": round(mrr_sum / n, 3),
|
|
"session_journal_queries_answered": f"{ts_answered}/{ts_total}"}
|
|
|
|
|
|
def eval_retrieval(h):
|
|
h.reset()
|
|
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
|
for path, text in {**ENTITY_CORPUS, **TIME_SERIES_CORPUS}.items():
|
|
h.seed(path, text)
|
|
r = h.run(SWEEP, "--apply")
|
|
assert r.returncode == 0, f"sweep failed: {r.stdout}{r.stderr}"
|
|
|
|
current_ranked, baseline_ranked = [], []
|
|
fresh_top1_current = fresh_top1_baseline = None
|
|
for q, gold, ts_only in QUERIES:
|
|
r = h.run(ECHO, "recall", q, "--json", "--limit", "5")
|
|
try:
|
|
primary = [hit["path"] for hit in json.loads(r.stdout).get("primary", [])]
|
|
except Exception: # noqa: BLE001
|
|
primary = []
|
|
current_ranked.append(((q, gold, ts_only), primary))
|
|
base = keyword_baseline(q, ENTITY_CORPUS)
|
|
baseline_ranked.append(((q, gold, ts_only), base))
|
|
if q == FRESHNESS_QUERY:
|
|
fresh_top1_current = primary[0] if primary else None
|
|
fresh_top1_baseline = base[0] if base else None
|
|
|
|
cur = metrics_for(current_ranked)
|
|
cur["freshness_top1_correct"] = fresh_top1_current == FRESHNESS_TOP1
|
|
bas = metrics_for(baseline_ranked)
|
|
bas["freshness_top1_correct"] = fresh_top1_baseline == FRESHNESS_TOP1
|
|
return {"current (hybrid+priors, full corpus)": cur,
|
|
"baseline (keyword, entities only)": bas}
|
|
|
|
|
|
# ------------------------------------------------------------------- 2. DEDUP
|
|
# Attempts referring to an EXISTING entity under another name; a create == duplicate.
|
|
DUP_ATTEMPTS = [
|
|
("Robert Smith", "person", "resources/people/robert-smith.md"),
|
|
("Echo Memory", "project", "projects/active/echo-memory.md"),
|
|
("Acme Consulting", "company", "resources/companies/acme-consulting.md"),
|
|
]
|
|
# Legitimate NEW entities; a block == false positive (must create in both modes).
|
|
LEGIT_ATTEMPTS = [
|
|
("Echo Review Meeting", "meeting",
|
|
f"resources/meetings/{TODAY}-echo-review-meeting.md"), # cross-kind lookalike
|
|
("Zephyr", "concept", "resources/concepts/zephyr.md"), # unrelated
|
|
]
|
|
|
|
|
|
def _dedup_pass(h, gate_env):
|
|
h.reset()
|
|
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
|
for title, kind in [("Bob Smith", "person"), ("Echo", "project"), ("Acme", "company")]:
|
|
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
|
dupes = blocked_legit = 0
|
|
for title, kind, would_be in DUP_ATTEMPTS:
|
|
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
|
dupes += int(h.ground(would_be) is not None)
|
|
for title, kind, path in LEGIT_ATTEMPTS:
|
|
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
|
blocked_legit += int(h.ground(path) is None)
|
|
return {"duplicate_notes_created": dupes, "legitimate_captures_blocked": blocked_legit,
|
|
"referring_attempts": len(DUP_ATTEMPTS), "legit_attempts": len(LEGIT_ATTEMPTS)}
|
|
|
|
|
|
def eval_dedup(h):
|
|
return {"gate ON (v1.5.1 default)": _dedup_pass(h, {}),
|
|
"gate OFF (pre-1.5, warn-only)": _dedup_pass(h, {"ECHO_DUP_GATE": "99"})}
|
|
|
|
|
|
# ------------------------------------------------------------- 3. WRITE SAFETY
|
|
def eval_write_safety(h):
|
|
h.reset()
|
|
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
|
silent = loud = 0
|
|
ops = 0
|
|
|
|
def bodyfile(text):
|
|
f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
|
|
f.write(text)
|
|
f.close()
|
|
return f.name
|
|
|
|
# transient 503 on first write ("flaky" marker) -> client must retry and land it
|
|
ops += 1
|
|
r = h.run(ECHO, "put", "resources/concepts/flaky-note.md", bodyfile("# Flaky\nok\n"))
|
|
landed = h.ground("resources/concepts/flaky-note.md") is not None
|
|
if r.returncode == 0 and not landed:
|
|
silent += 1
|
|
if r.returncode != 0:
|
|
loud += 1
|
|
|
|
# phantom PUT ("phantom" marker: 200 but not persisted) -> verify must fail LOUD
|
|
ops += 1
|
|
r = h.run(ECHO, "put", "resources/concepts/phantom-note.md", bodyfile("# Phantom\n"))
|
|
landed = h.ground("resources/concepts/phantom-note.md") is not None
|
|
if r.returncode == 0 and not landed:
|
|
silent += 1
|
|
if r.returncode != 0 and not landed:
|
|
loud += 1
|
|
|
|
# PATCH to a missing heading -> 400 must be surfaced, nothing written
|
|
ops += 1
|
|
h.seed("resources/concepts/target.md", "# Target\n\n## Real\nx\n")
|
|
r = h.run(ECHO, "patch", "resources/concepts/target.md", "append", "heading",
|
|
"Target::Nope", bodyfile("lost?"))
|
|
if r.returncode == 0 and "lost?" not in (h.ground("resources/concepts/target.md") or ""):
|
|
silent += 1
|
|
if r.returncode != 0:
|
|
loud += 1
|
|
|
|
# replayed append -> whole-line idempotency, no duplicate
|
|
ops += 1
|
|
h.run(ECHO, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
|
h.run(ECHO, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
|
n = (h.ground("inbox/captures/inbox.md") or "").count("replay me")
|
|
if n != 1:
|
|
silent += 1
|
|
|
|
return {"fault_scenarios": ops, "silent_write_failures": silent,
|
|
"failures_surfaced_loud": loud}
|
|
|
|
|
|
# --------------------------------------------------------------- 4. DURABILITY
|
|
def eval_durability(h, dead_base):
|
|
h.reset()
|
|
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
|
queued_ok = 0
|
|
for i in range(3):
|
|
r = h.run(ECHO, "append", "inbox/captures/inbox.md",
|
|
f"- {TODAY}: offline capture {i}", base=dead_base)
|
|
queued_ok += int(r.returncode == 0 and "queued" in r.stdout)
|
|
# vault returns: flush replays; a second flush must not duplicate
|
|
h.run(ECHO, "flush")
|
|
h.run(ECHO, "flush")
|
|
inbox = h.ground("inbox/captures/inbox.md") or ""
|
|
landed = sum(1 for i in range(3) if f"offline capture {i}" in inbox)
|
|
dupes = sum(inbox.count(f"offline capture {i}") - 1
|
|
for i in range(3) if f"offline capture {i}" in inbox)
|
|
return {"writes_during_outage": 3, "reported_queued": queued_ok,
|
|
"landed_after_flush": landed, "lost": 3 - landed, "duplicated_on_reflush": dupes}
|
|
|
|
|
|
# ------------------------------------------------------------------------ main
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--port", type=int, default=8797)
|
|
a = ap.parse_args()
|
|
base = f"http://127.0.0.1:{a.port}"
|
|
srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
|
state = tempfile.mkdtemp(prefix="echo-eval-state-")
|
|
try:
|
|
for _ in range(50):
|
|
try:
|
|
urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1)
|
|
break
|
|
except Exception: # noqa: BLE001
|
|
time.sleep(0.1)
|
|
h = Harness(base, state)
|
|
results = {
|
|
"version": "1.5.1", "date": TODAY,
|
|
"retrieval": eval_retrieval(h),
|
|
"dedup": eval_dedup(h),
|
|
"write_safety": eval_write_safety(h),
|
|
"durability": eval_durability(h, dead_base=f"http://127.0.0.1:{a.port + 1}"),
|
|
}
|
|
out = HERE / "results" / "latest.json"
|
|
out.parent.mkdir(exist_ok=True)
|
|
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
|
|
|
|
print(f"\n== echo-memory eval — v{results['version']} ({results['date']}) ==\n")
|
|
for section, data in results.items():
|
|
if section in ("version", "date"):
|
|
continue
|
|
print(f"## {section}")
|
|
for k, v in data.items():
|
|
if isinstance(v, dict):
|
|
print(f" {k}:")
|
|
for k2, v2 in v.items():
|
|
print(f" {k2:38} {v2}")
|
|
else:
|
|
print(f" {k:40} {v}")
|
|
print()
|
|
print(f"results -> {out}")
|
|
return 0
|
|
finally:
|
|
srv.terminate()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|