Retire ROADMAP-1.0; rewrite run_eval.py as current-version metrics harness
Knocks two items off ROADMAP-2.0 (§4 eval refresh, §5 roadmap retirement): - run_eval.py: the stale 0.6-vs-0.7 A/B is replaced (old version in git history) with a four-part current-version eval against the mock — retrieval (hybrid+priors vs keyword/entities-only baseline: R@5 1.00 vs 0.75, MRR 1.00 vs 0.75, session/journal queries 2/2 vs 0/2, freshness top-1 correct vs wrong), dedup (0 dupes gate-on vs 3 gate-off, 0 false blocks), write safety (0 silent failures across 4 fault scenarios), durability (3/3 offline writes queued+landed, 0 lost, 0 re-flush dupes). Results in eval/results/latest.json. - README: metrics table published; repo-layout eval descriptor updated. - eval/README: new harness documented; A/B framing marked historical. - ROADMAP-1.0.md removed (fully shipped; story lives in the README version table + git history); dangling docstring pointer fixed in echo_concurrency.py; ROADMAP-2.0 checkboxes ticked. All suites green; artifact rebuilt (1.5.1, docstring-only source change). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jason Stedwell
+19
-5
@@ -1,9 +1,23 @@
|
||||
# echo-memory eval — 0.6 vs 0.7 A/B harness
|
||||
# echo-memory eval — current-version metrics harness
|
||||
|
||||
A reproducible, credential-free A/B comparison of the plugin **before** (0.6: raw-curl
|
||||
recipes from `SKILL.md`) and **after** (the shipped `scripts/echo.py` client) the
|
||||
hardening work. It quantifies the claims in the comparative analysis: token cost of the
|
||||
I/O layer, and the rate of **silent write failures**.
|
||||
A reproducible, credential-free eval of the shipped plugin against the deterministic
|
||||
mock REST API. `run_eval.py` measures the four things the plugin claims (results print
|
||||
as a table and land in `results/latest.json`):
|
||||
|
||||
1. **Retrieval** — hybrid recall (BM25 over entities **and** sessions/journal, fused
|
||||
with freshness/status priors) vs a keyword-only baseline modeling pre-1.5 recall
|
||||
(per-word substring ranking over entity notes only). Precision@5 / recall@5 / MRR,
|
||||
session/journal answerability, freshness top-1.
|
||||
2. **Dedup** — duplicate notes created replaying a capture stream with the pre-write
|
||||
gate ON (default) vs OFF (`ECHO_DUP_GATE=99` ≈ pre-1.5 warn-only), plus legitimate
|
||||
captures wrongly blocked (must be 0).
|
||||
3. **Write safety** — silent write failures under fault injection (transient 503,
|
||||
phantom PUT, PATCH to a missing heading, replayed append). Must be 0; must be loud.
|
||||
4. **Durability** — writes during an outage queue, replay exactly once, never
|
||||
duplicate on re-flush.
|
||||
|
||||
> The original **0.6-vs-0.7 A/B harness** (token cost + silent-failure comparison that
|
||||
> motivated the 0.7 client) lives in git history — `git log -- eval/run_eval.py`.
|
||||
|
||||
## Current test suites (run these for any change)
|
||||
|
||||
|
||||
+39
-147
@@ -1,154 +1,46 @@
|
||||
{
|
||||
"params": {
|
||||
"port": 8799,
|
||||
"cpt": 4.0,
|
||||
"recovery": 1500,
|
||||
"detect_cost": 80
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"scenario": "agent-log-missing-heading",
|
||||
"fault": "bad heading -> 400 invalid-target",
|
||||
"06": {
|
||||
"emit_tokens": 82,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 22,
|
||||
"claimed_ok": false,
|
||||
"detected": true,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
"version": "1.5.1",
|
||||
"date": "2026-07-03",
|
||||
"retrieval": {
|
||||
"current (hybrid+priors, full corpus)": {
|
||||
"precision_at_5": 0.25,
|
||||
"recall_at_5": 1.0,
|
||||
"mrr": 1.0,
|
||||
"session_journal_queries_answered": "2/2",
|
||||
"freshness_top1_correct": true
|
||||
},
|
||||
{
|
||||
"scenario": "scope-switch",
|
||||
"fault": "(none)",
|
||||
"06": {
|
||||
"emit_tokens": 84,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 24,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "inbox-capture-replayed",
|
||||
"fault": "duplicate attempt (retry/replay)",
|
||||
"06": {
|
||||
"emit_tokens": 125,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 1,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 33,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "session-log-flaky-network",
|
||||
"fault": "one-time 503 (transient)",
|
||||
"06": {
|
||||
"emit_tokens": 64,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 17,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "heartbeat-phantom-write",
|
||||
"fault": "accepted-but-not-persisted",
|
||||
"06": {
|
||||
"emit_tokens": 61,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 14,
|
||||
"claimed_ok": false,
|
||||
"detected": true,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "cold-start-load-6-reads",
|
||||
"fault": "(none)",
|
||||
"06": {
|
||||
"emit_tokens": 192,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 64,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"totals": {
|
||||
"0.6": {
|
||||
"gen_tokens": 608,
|
||||
"silent_failures": 4,
|
||||
"detected_failures": 0,
|
||||
"duplicates": 1,
|
||||
"effective_tokens": 6608
|
||||
},
|
||||
"0.7": {
|
||||
"gen_tokens": 174,
|
||||
"silent_failures": 0,
|
||||
"detected_failures": 2,
|
||||
"duplicates": 0,
|
||||
"effective_tokens": 334
|
||||
"baseline (keyword, entities only)": {
|
||||
"precision_at_5": 0.2,
|
||||
"recall_at_5": 0.75,
|
||||
"mrr": 0.75,
|
||||
"session_journal_queries_answered": "0/2",
|
||||
"freshness_top1_correct": false
|
||||
}
|
||||
},
|
||||
"silent_error_free": {
|
||||
"0.6": "1/5",
|
||||
"0.7": "5/5"
|
||||
"dedup": {
|
||||
"gate ON (v1.5.1 default)": {
|
||||
"duplicate_notes_created": 0,
|
||||
"legitimate_captures_blocked": 0,
|
||||
"referring_attempts": 3,
|
||||
"legit_attempts": 2
|
||||
},
|
||||
"gate OFF (pre-1.5, warn-only)": {
|
||||
"duplicate_notes_created": 3,
|
||||
"legitimate_captures_blocked": 0,
|
||||
"referring_attempts": 3,
|
||||
"legit_attempts": 2
|
||||
}
|
||||
},
|
||||
"writes_persisted": {
|
||||
"0.6": "2/5",
|
||||
"0.7": "3/5"
|
||||
"write_safety": {
|
||||
"fault_scenarios": 4,
|
||||
"silent_write_failures": 0,
|
||||
"failures_surfaced_loud": 2
|
||||
},
|
||||
"durability": {
|
||||
"writes_during_outage": 3,
|
||||
"reported_queued": 3,
|
||||
"landed_after_flush": 3,
|
||||
"lost": 0,
|
||||
"duplicated_on_reflush": 0
|
||||
}
|
||||
}
|
||||
+321
-256
@@ -1,41 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""run_eval.py — A/B eval of echo-memory 0.6 (raw curl, documented recipes) vs
|
||||
0.7 (the shipped echo.py client) over representative memory operations, with
|
||||
fault injection.
|
||||
"""run_eval.py — current-version metrics for echo-memory (v1.5+).
|
||||
|
||||
Design
|
||||
------
|
||||
* A mock Obsidian REST API (mock_olrapi.py) gives deterministic behavior + faults,
|
||||
so no credentials are needed and the real vault is never touched.
|
||||
* The 0.7 side runs the ACTUAL shipped scripts/echo.py (status-checked, retry, verify,
|
||||
idempotent append).
|
||||
* The 0.6 side faithfully models the documented raw-curl recipes: it performs the
|
||||
same HTTP but does NOT inspect status, does NOT retry, does NOT verify, and does
|
||||
NOT dedupe — exactly what SKILL.md 0.6 told the model to emit.
|
||||
* Ground truth is read back independently from the mock after each method, so a
|
||||
"silent failure" (method reported success but the vault is actually wrong, and
|
||||
nobody noticed) is detected regardless of what the method claimed.
|
||||
* Each (scenario, method) runs against a freshly reset + re-seeded server, so faults
|
||||
(e.g. one-time 503) are identical for both methods.
|
||||
Replaces the historical 0.6-vs-0.7 A/B harness (see git history for that version).
|
||||
Measures the four things the plugin now claims, against the deterministic mock
|
||||
REST API (mock_olrapi.py) — no credentials, no live vault:
|
||||
|
||||
Metrics
|
||||
-------
|
||||
* gen_tokens : output tokens the MODEL must generate for the op (len(emitted)/CPT).
|
||||
* silent_failure : claimed success BUT ground truth is wrong (lost write or duplicate).
|
||||
* detected : the method surfaced the failure (loud, retryable) instead of hiding it.
|
||||
* effective_tokens = gen_tokens + silent_failures*RECOVERY + detected*DETECT_COST
|
||||
(RECOVERY/DETECT_COST are labeled assumptions, tune via env).
|
||||
1. RETRIEVAL — hybrid recall (BM25 over entities + sessions/journal, fused with
|
||||
freshness/status priors) vs a keyword-only baseline modeling the
|
||||
pre-1.5 recall (per-word substring ranking over ENTITY notes only).
|
||||
Metrics: precision@5, recall@5, MRR, session/journal queries
|
||||
answered, freshness top-1 correctness.
|
||||
2. DEDUP — duplicate notes created replaying a capture stream with the
|
||||
pre-write gate ON (default) vs OFF (ECHO_DUP_GATE=99, i.e. the
|
||||
pre-1.5 warn-only behavior). Also counts legitimate captures
|
||||
wrongly blocked (must be 0).
|
||||
3. WRITE SAFETY— silent write failures (claimed success but ground truth wrong)
|
||||
across fault injection: transient 503, phantom PUT, PATCH to a
|
||||
missing heading, replayed append. Must be 0; failures must be LOUD.
|
||||
4. DURABILITY — writes issued while the vault is unreachable must queue, replay
|
||||
exactly once when it returns, and never duplicate on re-flush.
|
||||
|
||||
Usage: python3 run_eval.py [--port 8799] [--cpt 4] [--recovery 1500] [--detect-cost 80]
|
||||
Ground truth is always read back independently via the mock's __debug__ endpoint.
|
||||
Results print as a table and land in results/latest.json.
|
||||
|
||||
Usage: python3 run_eval.py [--port 8797]
|
||||
"""
|
||||
import os, sys, json, time, subprocess, tempfile, argparse, urllib.request, urllib.error
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"
|
||||
ECHO = SCRIPTS / "echo.py"
|
||||
SWEEP = SCRIPTS / "sweep.py"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
TODAY = "2026-07-03"
|
||||
|
||||
|
||||
# ----- tiny HTTP helpers (used for setup, ground truth, and the 0.6 model) -----
|
||||
def http(method, url, body=None, headers=None):
|
||||
data = body.encode() if isinstance(body, str) else body
|
||||
req = urllib.request.Request(url, data=data, method=method,
|
||||
@@ -45,259 +56,313 @@ def http(method, url, body=None, headers=None):
|
||||
return r.status, r.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.read().decode("utf-8", "replace")
|
||||
except Exception as e:
|
||||
except Exception as e: # noqa: BLE001
|
||||
return 0, str(e)
|
||||
|
||||
class Eval:
|
||||
def __init__(self, base, cpt):
|
||||
self.base, self.cpt = base, cpt
|
||||
def reset(self): http("POST", f"{self.base}/__debug__reset")
|
||||
def seed(self, path, content): http("PUT", f"{self.base}/vault/{path}", content)
|
||||
|
||||
class Harness:
|
||||
def __init__(self, base, state_dir):
|
||||
self.base = base
|
||||
self.state_dir = state_dir
|
||||
|
||||
def reset(self):
|
||||
http("POST", f"{self.base}/__debug__reset")
|
||||
|
||||
def seed(self, path, content):
|
||||
http("PUT", f"{self.base}/vault/{path}", content)
|
||||
|
||||
def ground(self, path):
|
||||
st, body = http("GET", f"{self.base}/__debug__?path={path}")
|
||||
_, body = http("GET", f"{self.base}/__debug__?path={path}")
|
||||
return None if body == "<<MISSING>>" else body
|
||||
def toks(self, text): return round(len(text) / self.cpt)
|
||||
def echo(self, *args):
|
||||
env = dict(os.environ, ECHO_BASE=self.base, ECHO_KEY=KEY, ECHO_VERIFY="1", ECHO_LOCK_TTL="900")
|
||||
p = subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env)
|
||||
return p.returncode
|
||||
|
||||
# ---- faithful 0.6 recipe text (what the model emitted), for token accounting ----
|
||||
def recipe_get(path):
|
||||
return (f'curl -s -H "Authorization: Bearer {KEY}" '
|
||||
f'"https://echoapi.alwisp.com/vault/{path}"')
|
||||
def recipe_put(path):
|
||||
return (f"cat > /tmp/obs_file.md << 'EOF'\n<body>\nEOF\n\n"
|
||||
f'curl -s -X PUT -H "Authorization: Bearer {KEY}" '
|
||||
f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_file.md '
|
||||
f'"https://echoapi.alwisp.com/vault/{path}"')
|
||||
def recipe_post(path):
|
||||
return (f"cat > /tmp/obs_entry.md << 'EOF'\n<line>\nEOF\n\n"
|
||||
f'curl -s -X POST -H "Authorization: Bearer {KEY}" '
|
||||
f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_entry.md '
|
||||
f'"https://echoapi.alwisp.com/vault/{path}"')
|
||||
def recipe_patch(path, target):
|
||||
return (f"cat > /tmp/obs_patch.md << 'EOF'\n<body>\nEOF\n\n"
|
||||
f'curl -s -X PATCH -H "Authorization: Bearer {KEY}" '
|
||||
f'-H "Operation: append" -H "Target-Type: heading" -H "Target: {target}" '
|
||||
f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_patch.md '
|
||||
f'"https://echoapi.alwisp.com/vault/{path}"')
|
||||
def run(self, script, *args, env_extra=None, base=None):
|
||||
env = dict(os.environ, ECHO_BASE=base or self.base, ECHO_KEY=KEY,
|
||||
ECHO_VERIFY="1", ECHO_TODAY=TODAY, ECHO_STATE_DIR=self.state_dir,
|
||||
**(env_extra or {}))
|
||||
return subprocess.run([sys.executable, str(script), *args],
|
||||
capture_output=True, text=True, env=env)
|
||||
|
||||
def tmpfile(content):
|
||||
f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False); f.write(content); f.close()
|
||||
return f.name
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenarios. Each defines seed(), and a run for each method that returns:
|
||||
# {emit: <text model generates>, claimed_ok: bool, detected: bool}
|
||||
# plus a ground-truth check producing {persisted, duplicates}.
|
||||
# ---------------------------------------------------------------------------
|
||||
def scenarios(ev):
|
||||
out = []
|
||||
def note(fm_type, title, body, status="active", created=TODAY, updated=TODAY, tags=None):
|
||||
tags_s = "[" + ", ".join(tags or [fm_type]) + "]"
|
||||
return (f"---\ntype: {fm_type}\nstatus: {status}\ncreated: {created}\n"
|
||||
f"updated: {updated}\ntags: {tags_s}\nagent_written: true\nsource_notes: []\n---\n\n"
|
||||
f"# {title}\n\n{body}\n\n## Related\n")
|
||||
|
||||
# S1 — agent-log append where the heading is MISSING (-> 400 invalid-target)
|
||||
def s1():
|
||||
path, tgt = "journal/daily/2026-06-19.md", "2026-06-19::Agent Log"
|
||||
line = "- 2026-06-19: EVALMARK-s1"
|
||||
def seed(): ev.seed(path, "---\ntype: daily-note\n---\n\n# 2026-06-19\n\n## Notes\n")
|
||||
def m06():
|
||||
http("PATCH", f"{ev.base}/vault/{path}", line,
|
||||
{"Operation": "append", "Target-Type": "heading", "Target": tgt}) # status ignored
|
||||
return {"emit": recipe_patch(path, tgt), "claimed_ok": True, "detected": False}
|
||||
def m07():
|
||||
rc = ev.echo("patch", path, "append", "heading", tgt, tmpfile(line))
|
||||
return {"emit": f'"$ECHO" patch {path} append heading "{tgt}" /tmp/x.md',
|
||||
"claimed_ok": rc == 0, "detected": rc != 0}
|
||||
def gt():
|
||||
c = ev.ground(path) or ""
|
||||
return {"persisted": "EVALMARK-s1" in c, "duplicates": max(0, c.count("EVALMARK-s1") - 1)}
|
||||
return dict(name="agent-log-missing-heading", fault="bad heading -> 400 invalid-target",
|
||||
seed=seed, m06=m06, m07=m07, gt=gt)
|
||||
out.append(s1())
|
||||
|
||||
# S2 — scope switch (no fault; pure token comparison on a PATCH replace)
|
||||
def s2():
|
||||
path, tgt = "_agent/context/current-context.md", "Current Context::Scope"
|
||||
def seed(): ev.seed(path, "---\ntype: context-bundle\n---\n\n# Current Context\n\n## Scope\nold scope\n")
|
||||
def m06():
|
||||
http("PATCH", f"{ev.base}/vault/{path}", "EVALMARK-s2 new scope",
|
||||
{"Operation": "replace", "Target-Type": "heading", "Target": tgt})
|
||||
return {"emit": recipe_patch(path, tgt), "claimed_ok": True, "detected": False}
|
||||
def m07():
|
||||
rc = ev.echo("patch", path, "replace", "heading", tgt, tmpfile("EVALMARK-s2 new scope"))
|
||||
return {"emit": f'"$ECHO" patch {path} replace heading "{tgt}" /tmp/x.md',
|
||||
"claimed_ok": rc == 0, "detected": rc != 0}
|
||||
def gt():
|
||||
c = ev.ground(path) or ""
|
||||
return {"persisted": "EVALMARK-s2" in c, "duplicates": 0}
|
||||
return dict(name="scope-switch", fault="(none)", seed=seed, m06=m06, m07=m07, gt=gt)
|
||||
out.append(s2())
|
||||
# ---------------------------------------------------------------- 1. RETRIEVAL
|
||||
ENTITY_CORPUS = {
|
||||
"resources/people/maria-lopez.md": note(
|
||||
"person", "Maria Lopez", "Runs procurement at Vantage Systems. Prefers email over calls."),
|
||||
"resources/companies/vantage-systems.md": note(
|
||||
"company", "Vantage Systems", "Enterprise procurement platform vendor; the contract renews in Q3."),
|
||||
"resources/concepts/bm25-ranking.md": note(
|
||||
"concept", "BM25 Ranking", "Lexical retrieval scoring with term frequency saturation and document length normalization."),
|
||||
"resources/concepts/graph-expansion.md": note(
|
||||
"concept", "Graph Expansion", "Expanding a hit into its linked neighbourhood along Related links and source notes."),
|
||||
"projects/active/website-redesign.md": note(
|
||||
"project", "Website Redesign",
|
||||
"Rebuild the marketing site. Navigation cleanup and new hero copy.",
|
||||
status="active", created="2026-06-25", updated="2026-07-01"),
|
||||
"projects/archived/old-website.md": note(
|
||||
"project", "Old Website",
|
||||
"Legacy marketing site. Navigation used nested dropdown menus. Navigation was the top complaint.",
|
||||
status="archived", created="2024-03-01", updated="2024-08-10"),
|
||||
"_agent/memory/semantic/tooling-preferences.md": note(
|
||||
"semantic-memory", "Tooling Preferences",
|
||||
"The operator prefers uv over pip for Python tooling, and ruff for linting."),
|
||||
}
|
||||
TIME_SERIES_CORPUS = {
|
||||
"_agent/sessions/2026-06-20-1000-vantage-contract-review.md": note(
|
||||
"session-log", "Vantage contract review",
|
||||
"Reviewed the Vantage contract. Decided to renegotiate the SLA penalty clause before renewal.",
|
||||
status="complete", created="2026-06-20", updated="2026-06-20", tags=["session-log"]),
|
||||
"journal/daily/2026-06-20.md": (
|
||||
"---\ntype: daily-note\ncreated: 2026-06-20\nupdated: 2026-06-20\ntags: [daily]\n---\n\n"
|
||||
"# 2026-06-20\n\n## Agent Log\n- 2026-06-20: renegotiation kickoff email sent to Vantage\n"),
|
||||
}
|
||||
|
||||
# S3 — inbox capture issued TWICE (retry/replay): dedup vs duplicate line
|
||||
def s3():
|
||||
path = "inbox/captures/inbox.md"; line = "- 2026-06-19: EVALMARK-s3"
|
||||
def seed(): ev.seed(path, "# Inbox\n")
|
||||
def m06():
|
||||
http("POST", f"{ev.base}/vault/{path}", line + "\n") # no idempotency check
|
||||
http("POST", f"{ev.base}/vault/{path}", line + "\n")
|
||||
return {"emit": recipe_post(path) + "\n# (repeated on retry)\n" + recipe_post(path),
|
||||
"claimed_ok": True, "detected": False}
|
||||
def m07():
|
||||
rc1 = ev.echo("append", path, line)
|
||||
rc2 = ev.echo("append", path, line) # second is skipped by read-before-POST
|
||||
return {"emit": f'"$ECHO" append {path} "{line}"\n"$ECHO" append {path} "{line}"',
|
||||
"claimed_ok": rc1 == 0 and rc2 == 0, "detected": False}
|
||||
def gt():
|
||||
c = ev.ground(path) or ""
|
||||
return {"persisted": "EVALMARK-s3" in c, "duplicates": max(0, c.count("EVALMARK-s3") - 1)}
|
||||
return dict(name="inbox-capture-replayed", fault="duplicate attempt (retry/replay)",
|
||||
seed=seed, m06=m06, m07=m07, gt=gt)
|
||||
out.append(s3())
|
||||
# (query, gold paths, only answerable from sessions/journal)
|
||||
QUERIES = [
|
||||
("uv or pip for python tooling", {"_agent/memory/semantic/tooling-preferences.md"}, False),
|
||||
("procurement contact", {"resources/people/maria-lopez.md",
|
||||
"resources/companies/vantage-systems.md"}, False),
|
||||
("SLA penalty clause decision",
|
||||
{"_agent/sessions/2026-06-20-1000-vantage-contract-review.md"}, True),
|
||||
("marketing site navigation", {"projects/active/website-redesign.md",
|
||||
"projects/archived/old-website.md"}, False),
|
||||
("term frequency saturation", {"resources/concepts/bm25-ranking.md"}, False),
|
||||
("contract renewal Q3", {"resources/companies/vantage-systems.md"}, False),
|
||||
("renegotiation kickoff", {"journal/daily/2026-06-20.md"}, True),
|
||||
("graph neighbourhood expansion", {"resources/concepts/graph-expansion.md"}, False),
|
||||
]
|
||||
FRESHNESS_QUERY = "marketing site navigation"
|
||||
FRESHNESS_TOP1 = "projects/active/website-redesign.md"
|
||||
|
||||
# S4 — session-log PUT under a flaky network (one-time 503 then success)
|
||||
def s4():
|
||||
path = "_agent/sessions/2026-06-19-1430-flaky-eval.md"
|
||||
body = "---\ntype: session-log\n---\n\n# Session\nEVALMARK-s4\n"
|
||||
def seed(): pass # file does not pre-exist
|
||||
def m06():
|
||||
http("PUT", f"{ev.base}/vault/{path}", body) # single shot, no retry, status ignored
|
||||
return {"emit": recipe_put(path), "claimed_ok": True, "detected": False}
|
||||
def m07():
|
||||
rc = ev.echo("put", path, tmpfile(body)) # echo.py retries the 503 once
|
||||
return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0}
|
||||
def gt():
|
||||
c = ev.ground(path) or ""
|
||||
return {"persisted": "EVALMARK-s4" in c, "duplicates": 0}
|
||||
return dict(name="session-log-flaky-network", fault="one-time 503 (transient)",
|
||||
seed=seed, m06=m06, m07=m07, gt=gt)
|
||||
out.append(s4())
|
||||
|
||||
# S5 — heartbeat PUT accepted-but-not-persisted (proxy hiccup): verify catches it
|
||||
def s5():
|
||||
path = "_agent/heartbeat/phantom-eval.md"; body = "EVALMARK-s5 @ 2026-06-19T14:30:00Z\n"
|
||||
def seed(): pass
|
||||
def m06():
|
||||
http("PUT", f"{ev.base}/vault/{path}", body) # 200 returned; no read-back verify
|
||||
return {"emit": recipe_put(path), "claimed_ok": True, "detected": False}
|
||||
def m07():
|
||||
rc = ev.echo("put", path, tmpfile(body)) # verify GET -> 404 -> die
|
||||
return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0}
|
||||
def gt():
|
||||
c = ev.ground(path) or ""
|
||||
return {"persisted": "EVALMARK-s5" in c, "duplicates": 0}
|
||||
return dict(name="heartbeat-phantom-write", fault="accepted-but-not-persisted",
|
||||
seed=seed, m06=m06, m07=m07, gt=gt)
|
||||
out.append(s5())
|
||||
def keyword_baseline(query, corpus, k=5):
|
||||
"""Models pre-1.5 keyword recall: per-word substring counting over ENTITY note
|
||||
bodies only (sessions/journal were not in the corpus before 1.5)."""
|
||||
words = [w for w in re.findall(r"[a-z0-9]+", query.lower()) if len(w) > 2]
|
||||
scored = []
|
||||
for path, text in corpus.items():
|
||||
body = text.lower()
|
||||
s = sum(body.count(w) for w in words)
|
||||
if s > 0:
|
||||
scored.append((s, path))
|
||||
scored.sort(key=lambda x: (-x[0], x[1]))
|
||||
return [p for _, p in scored[:k]]
|
||||
|
||||
# S6 — cold-start load: 6 reads (no fault; pure token comparison)
|
||||
def s6():
|
||||
paths = ["_agent/echo-vault.md", "_agent/memory/semantic/operator-preferences.md",
|
||||
"_agent/context/current-context.md", "_agent/heartbeat/last-session.md",
|
||||
"journal/daily/2026-06-19.md", "inbox/captures/inbox.md"]
|
||||
def seed():
|
||||
for p in paths: ev.seed(p, f"# {p}\nEVALMARK-s6\n")
|
||||
def m06():
|
||||
for p in paths: http("GET", f"{ev.base}/vault/{p}")
|
||||
return {"emit": "\n".join(recipe_get(p) for p in paths), "claimed_ok": True, "detected": False}
|
||||
def m07():
|
||||
for p in paths: ev.echo("get", p)
|
||||
return {"emit": "\n".join(f'"$ECHO" get {p}' for p in paths), "claimed_ok": True, "detected": False}
|
||||
def gt(): return {"persisted": True, "duplicates": 0}
|
||||
return dict(name="cold-start-load-6-reads", fault="(none)", seed=seed, m06=m06, m07=m07, gt=gt)
|
||||
out.append(s6())
|
||||
|
||||
return out
|
||||
def metrics_for(ranked_by_query):
|
||||
p_sum = r_sum = mrr_sum = 0.0
|
||||
ts_answered = ts_total = 0
|
||||
for (q, gold, ts_only), ranked in ranked_by_query:
|
||||
top = ranked[:5]
|
||||
hits = [p for p in top if p in gold]
|
||||
p_sum += len(hits) / 5
|
||||
r_sum += len(hits) / len(gold)
|
||||
rr = 0.0
|
||||
for i, p in enumerate(top):
|
||||
if p in gold:
|
||||
rr = 1.0 / (i + 1)
|
||||
break
|
||||
mrr_sum += rr
|
||||
if ts_only:
|
||||
ts_total += 1
|
||||
ts_answered += int(bool(hits))
|
||||
n = len(ranked_by_query)
|
||||
return {"precision_at_5": round(p_sum / n, 3), "recall_at_5": round(r_sum / n, 3),
|
||||
"mrr": round(mrr_sum / n, 3),
|
||||
"session_journal_queries_answered": f"{ts_answered}/{ts_total}"}
|
||||
|
||||
def run_method(ev, scn, key):
|
||||
ev.reset(); scn["seed"]()
|
||||
res = scn[key]()
|
||||
g = scn["gt"]()
|
||||
# a write op "should persist"; reads (s6) and replays are judged by their own gt
|
||||
bad = (not g["persisted"] and scn["name"] != "cold-start-load-6-reads") or g["duplicates"] > 0
|
||||
silent = bool(res["claimed_ok"] and bad)
|
||||
return {"emit_tokens": ev.toks(res["emit"]), "claimed_ok": res["claimed_ok"],
|
||||
"detected": res["detected"], "persisted": g["persisted"],
|
||||
"duplicates": g["duplicates"], "silent_failure": silent}
|
||||
|
||||
def eval_retrieval(h):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
for path, text in {**ENTITY_CORPUS, **TIME_SERIES_CORPUS}.items():
|
||||
h.seed(path, text)
|
||||
r = h.run(SWEEP, "--apply")
|
||||
assert r.returncode == 0, f"sweep failed: {r.stdout}{r.stderr}"
|
||||
|
||||
current_ranked, baseline_ranked = [], []
|
||||
fresh_top1_current = fresh_top1_baseline = None
|
||||
for q, gold, ts_only in QUERIES:
|
||||
r = h.run(ECHO, "recall", q, "--json", "--limit", "5")
|
||||
try:
|
||||
primary = [hit["path"] for hit in json.loads(r.stdout).get("primary", [])]
|
||||
except Exception: # noqa: BLE001
|
||||
primary = []
|
||||
current_ranked.append(((q, gold, ts_only), primary))
|
||||
base = keyword_baseline(q, ENTITY_CORPUS)
|
||||
baseline_ranked.append(((q, gold, ts_only), base))
|
||||
if q == FRESHNESS_QUERY:
|
||||
fresh_top1_current = primary[0] if primary else None
|
||||
fresh_top1_baseline = base[0] if base else None
|
||||
|
||||
cur = metrics_for(current_ranked)
|
||||
cur["freshness_top1_correct"] = fresh_top1_current == FRESHNESS_TOP1
|
||||
bas = metrics_for(baseline_ranked)
|
||||
bas["freshness_top1_correct"] = fresh_top1_baseline == FRESHNESS_TOP1
|
||||
return {"current (hybrid+priors, full corpus)": cur,
|
||||
"baseline (keyword, entities only)": bas}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- 2. DEDUP
|
||||
# Attempts referring to an EXISTING entity under another name; a create == duplicate.
|
||||
DUP_ATTEMPTS = [
|
||||
("Robert Smith", "person", "resources/people/robert-smith.md"),
|
||||
("Echo Memory", "project", "projects/active/echo-memory.md"),
|
||||
("Acme Consulting", "company", "resources/companies/acme-consulting.md"),
|
||||
]
|
||||
# Legitimate NEW entities; a block == false positive (must create in both modes).
|
||||
LEGIT_ATTEMPTS = [
|
||||
("Echo Review Meeting", "meeting",
|
||||
f"resources/meetings/{TODAY}-echo-review-meeting.md"), # cross-kind lookalike
|
||||
("Zephyr", "concept", "resources/concepts/zephyr.md"), # unrelated
|
||||
]
|
||||
|
||||
|
||||
def _dedup_pass(h, gate_env):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
for title, kind in [("Bob Smith", "person"), ("Echo", "project"), ("Acme", "company")]:
|
||||
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
dupes = blocked_legit = 0
|
||||
for title, kind, would_be in DUP_ATTEMPTS:
|
||||
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
dupes += int(h.ground(would_be) is not None)
|
||||
for title, kind, path in LEGIT_ATTEMPTS:
|
||||
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
blocked_legit += int(h.ground(path) is None)
|
||||
return {"duplicate_notes_created": dupes, "legitimate_captures_blocked": blocked_legit,
|
||||
"referring_attempts": len(DUP_ATTEMPTS), "legit_attempts": len(LEGIT_ATTEMPTS)}
|
||||
|
||||
|
||||
def eval_dedup(h):
|
||||
return {"gate ON (v1.5.1 default)": _dedup_pass(h, {}),
|
||||
"gate OFF (pre-1.5, warn-only)": _dedup_pass(h, {"ECHO_DUP_GATE": "99"})}
|
||||
|
||||
|
||||
# ------------------------------------------------------------- 3. WRITE SAFETY
|
||||
def eval_write_safety(h):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
silent = loud = 0
|
||||
ops = 0
|
||||
|
||||
def bodyfile(text):
|
||||
f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
|
||||
f.write(text)
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
# transient 503 on first write ("flaky" marker) -> client must retry and land it
|
||||
ops += 1
|
||||
r = h.run(ECHO, "put", "resources/concepts/flaky-note.md", bodyfile("# Flaky\nok\n"))
|
||||
landed = h.ground("resources/concepts/flaky-note.md") is not None
|
||||
if r.returncode == 0 and not landed:
|
||||
silent += 1
|
||||
if r.returncode != 0:
|
||||
loud += 1
|
||||
|
||||
# phantom PUT ("phantom" marker: 200 but not persisted) -> verify must fail LOUD
|
||||
ops += 1
|
||||
r = h.run(ECHO, "put", "resources/concepts/phantom-note.md", bodyfile("# Phantom\n"))
|
||||
landed = h.ground("resources/concepts/phantom-note.md") is not None
|
||||
if r.returncode == 0 and not landed:
|
||||
silent += 1
|
||||
if r.returncode != 0 and not landed:
|
||||
loud += 1
|
||||
|
||||
# PATCH to a missing heading -> 400 must be surfaced, nothing written
|
||||
ops += 1
|
||||
h.seed("resources/concepts/target.md", "# Target\n\n## Real\nx\n")
|
||||
r = h.run(ECHO, "patch", "resources/concepts/target.md", "append", "heading",
|
||||
"Target::Nope", bodyfile("lost?"))
|
||||
if r.returncode == 0 and "lost?" not in (h.ground("resources/concepts/target.md") or ""):
|
||||
silent += 1
|
||||
if r.returncode != 0:
|
||||
loud += 1
|
||||
|
||||
# replayed append -> whole-line idempotency, no duplicate
|
||||
ops += 1
|
||||
h.run(ECHO, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
||||
h.run(ECHO, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
||||
n = (h.ground("inbox/captures/inbox.md") or "").count("replay me")
|
||||
if n != 1:
|
||||
silent += 1
|
||||
|
||||
return {"fault_scenarios": ops, "silent_write_failures": silent,
|
||||
"failures_surfaced_loud": loud}
|
||||
|
||||
|
||||
# --------------------------------------------------------------- 4. DURABILITY
|
||||
def eval_durability(h, dead_base):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
queued_ok = 0
|
||||
for i in range(3):
|
||||
r = h.run(ECHO, "append", "inbox/captures/inbox.md",
|
||||
f"- {TODAY}: offline capture {i}", base=dead_base)
|
||||
queued_ok += int(r.returncode == 0 and "queued" in r.stdout)
|
||||
# vault returns: flush replays; a second flush must not duplicate
|
||||
h.run(ECHO, "flush")
|
||||
h.run(ECHO, "flush")
|
||||
inbox = h.ground("inbox/captures/inbox.md") or ""
|
||||
landed = sum(1 for i in range(3) if f"offline capture {i}" in inbox)
|
||||
dupes = sum(inbox.count(f"offline capture {i}") - 1
|
||||
for i in range(3) if f"offline capture {i}" in inbox)
|
||||
return {"writes_during_outage": 3, "reported_queued": queued_ok,
|
||||
"landed_after_flush": landed, "lost": 3 - landed, "duplicated_on_reflush": dupes}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------ main
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=8799)
|
||||
ap.add_argument("--cpt", type=float, default=4.0, help="chars per token")
|
||||
ap.add_argument("--recovery", type=int, default=1500, help="token penalty per silent failure (recovery loop)")
|
||||
ap.add_argument("--detect-cost", type=int, default=80, help="token cost to re-issue a corrected call after a detected failure")
|
||||
ap.add_argument("--port", type=int, default=8797)
|
||||
a = ap.parse_args()
|
||||
|
||||
base = f"http://127.0.0.1:{a.port}"
|
||||
srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
state = tempfile.mkdtemp(prefix="echo-eval-state-")
|
||||
try:
|
||||
for _ in range(50): # wait for ready
|
||||
for _ in range(50):
|
||||
try:
|
||||
urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break
|
||||
except Exception: time.sleep(0.1)
|
||||
ev = Eval(base, a.cpt)
|
||||
rows, agg = [], {"06": {}, "07": {}}
|
||||
for scn in scenarios(ev):
|
||||
r06 = run_method(ev, scn, "m06")
|
||||
r07 = run_method(ev, scn, "m07")
|
||||
rows.append({"scenario": scn["name"], "fault": scn["fault"], "06": r06, "07": r07})
|
||||
urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1)
|
||||
break
|
||||
except Exception: # noqa: BLE001
|
||||
time.sleep(0.1)
|
||||
h = Harness(base, state)
|
||||
results = {
|
||||
"version": "1.5.1", "date": TODAY,
|
||||
"retrieval": eval_retrieval(h),
|
||||
"dedup": eval_dedup(h),
|
||||
"write_safety": eval_write_safety(h),
|
||||
"durability": eval_durability(h, dead_base=f"http://127.0.0.1:{a.port + 1}"),
|
||||
}
|
||||
out = HERE / "results" / "latest.json"
|
||||
out.parent.mkdir(exist_ok=True)
|
||||
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
|
||||
|
||||
def totals(side):
|
||||
gen = sum(r[side]["emit_tokens"] for r in rows)
|
||||
sil = sum(1 for r in rows if r[side]["silent_failure"])
|
||||
det = sum(1 for r in rows if r[side]["detected"])
|
||||
dup = sum(r[side]["duplicates"] for r in rows)
|
||||
eff = gen + sil * a.recovery + det * a.detect_cost
|
||||
return dict(gen_tokens=gen, silent_failures=sil, detected_failures=det,
|
||||
duplicates=dup, effective_tokens=eff)
|
||||
T06, T07 = totals("06"), totals("07")
|
||||
|
||||
# ---- report ----
|
||||
line = "=" * 78
|
||||
print(f"\n{line}\nECHO MEMORY — 0.6 vs 0.7 A/B EVAL")
|
||||
print(f"(mock OLRAPI; chars/token={a.cpt}, recovery-penalty={a.recovery}, detect-cost={a.detect_cost})\n{line}")
|
||||
hdr = f"{'scenario':28} {'fault':32} {'gen06':>5} {'gen07':>5} {'silent06':>8} {'silent07':>8}"
|
||||
print(hdr); print("-" * len(hdr))
|
||||
for r in rows:
|
||||
print(f"{r['scenario']:28} {r['fault']:32} "
|
||||
f"{r['06']['emit_tokens']:>5} {r['07']['emit_tokens']:>5} "
|
||||
f"{('YES' if r['06']['silent_failure'] else '-'):>8} "
|
||||
f"{('YES' if r['07']['silent_failure'] else '-'):>8}")
|
||||
print("-" * len(hdr))
|
||||
|
||||
def pct(old, new): return f"{(old-new)/old*100:+.0f}%" if old else "n/a"
|
||||
print(f"\n{'metric':30} {'0.6':>10} {'0.7':>10} {'delta':>10}")
|
||||
print("-" * 62)
|
||||
for label, k in [("generated tokens", "gen_tokens"),
|
||||
("silent failures", "silent_failures"),
|
||||
("duplicate lines", "duplicates"),
|
||||
("detected (loud) failures", "detected_failures"),
|
||||
("effective tokens (incl. recovery)", "effective_tokens")]:
|
||||
o, n = T06[k], T07[k]
|
||||
d = pct(o, n) if "token" in k else f"{n-o:+d}"
|
||||
print(f"{label:30} {o:>10} {n:>10} {d:>10}")
|
||||
wrows = [r for r in rows if r['scenario'] != 'cold-start-load-6-reads']
|
||||
denom = len(wrows)
|
||||
sf06 = sum(1 for r in wrows if not r['06']['silent_failure']) # silent-error-free
|
||||
sf07 = sum(1 for r in wrows if not r['07']['silent_failure'])
|
||||
pr06 = sum(1 for r in wrows if r['06']['persisted']) # write actually landed
|
||||
pr07 = sum(1 for r in wrows if r['07']['persisted'])
|
||||
print("-" * 62)
|
||||
print(f"{'silent-error-free ops':30} {str(sf06)+'/'+str(denom):>10} {str(sf07)+'/'+str(denom):>10}")
|
||||
print(f"{'writes actually persisted':30} {str(pr06)+'/'+str(denom):>10} {str(pr07)+'/'+str(denom):>10}")
|
||||
print(f" (the 2 un-persisted 0.7 ops are bad-heading + phantom: not persistable in a single")
|
||||
print(f" call, but 0.7 fails LOUD so the agent's ensure-heading/retry path can recover.)")
|
||||
print(f"\nNOTE: recovery-penalty and detect-cost are tunable ASSUMPTIONS, not measured.")
|
||||
print(f" gen tokens are a chars/{a.cpt} proxy for output tokens of the I/O layer only.")
|
||||
print(f" This harness measures mechanics, not model reasoning quality.\n")
|
||||
|
||||
report = {"params": vars(a), "rows": rows, "totals": {"0.6": T06, "0.7": T07},
|
||||
"silent_error_free": {"0.6": f"{sf06}/{denom}", "0.7": f"{sf07}/{denom}"},
|
||||
"writes_persisted": {"0.6": f"{pr06}/{denom}", "0.7": f"{pr07}/{denom}"}}
|
||||
(HERE / "results" / "latest.json").write_text(json.dumps(report, indent=2))
|
||||
print(f"wrote {HERE/'results'/'latest.json'}")
|
||||
print(f"\n== echo-memory eval — v{results['version']} ({results['date']}) ==\n")
|
||||
for section, data in results.items():
|
||||
if section in ("version", "date"):
|
||||
continue
|
||||
print(f"## {section}")
|
||||
for k, v in data.items():
|
||||
if isinstance(v, dict):
|
||||
print(f" {k}:")
|
||||
for k2, v2 in v.items():
|
||||
print(f" {k2:38} {v2}")
|
||||
else:
|
||||
print(f" {k:40} {v}")
|
||||
print()
|
||||
print(f"results -> {out}")
|
||||
return 0
|
||||
finally:
|
||||
srv.terminate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
raise SystemExit(main())
|
||||
|
||||
Reference in New Issue
Block a user