1
0
forked from jason/echo

Retire ROADMAP-1.0; rewrite run_eval.py as current-version metrics harness

Knocks two items off ROADMAP-2.0 (§4 eval refresh, §5 roadmap retirement):

- run_eval.py: the stale 0.6-vs-0.7 A/B is replaced (old version in git
  history) with a four-part current-version eval against the mock —
  retrieval (hybrid+priors vs keyword/entities-only baseline: R@5 1.00
  vs 0.75, MRR 1.00 vs 0.75, session/journal queries 2/2 vs 0/2,
  freshness top-1 correct vs wrong), dedup (0 dupes gate-on vs 3
  gate-off, 0 false blocks), write safety (0 silent failures across 4
  fault scenarios), durability (3/3 offline writes queued+landed, 0
  lost, 0 re-flush dupes). Results in eval/results/latest.json.
- README: metrics table published; repo-layout eval descriptor updated.
- eval/README: new harness documented; A/B framing marked historical.
- ROADMAP-1.0.md removed (fully shipped; story lives in the README
  version table + git history); dangling docstring pointer fixed in
  echo_concurrency.py; ROADMAP-2.0 checkboxes ticked.

All suites green; artifact rebuilt (1.5.1, docstring-only source change).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jason Stedwell
2026-07-03 13:30:01 -05:00
parent 919e411136
commit 993abdc846
9 changed files with 405 additions and 595 deletions
+39 -147
View File
@@ -1,154 +1,46 @@
{
"params": {
"port": 8799,
"cpt": 4.0,
"recovery": 1500,
"detect_cost": 80
},
"rows": [
{
"scenario": "agent-log-missing-heading",
"fault": "bad heading -> 400 invalid-target",
"06": {
"emit_tokens": 82,
"claimed_ok": true,
"detected": false,
"persisted": false,
"duplicates": 0,
"silent_failure": true
},
"07": {
"emit_tokens": 22,
"claimed_ok": false,
"detected": true,
"persisted": false,
"duplicates": 0,
"silent_failure": false
}
"version": "1.5.1",
"date": "2026-07-03",
"retrieval": {
"current (hybrid+priors, full corpus)": {
"precision_at_5": 0.25,
"recall_at_5": 1.0,
"mrr": 1.0,
"session_journal_queries_answered": "2/2",
"freshness_top1_correct": true
},
{
"scenario": "scope-switch",
"fault": "(none)",
"06": {
"emit_tokens": 84,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 0,
"silent_failure": false
},
"07": {
"emit_tokens": 24,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 0,
"silent_failure": false
}
},
{
"scenario": "inbox-capture-replayed",
"fault": "duplicate attempt (retry/replay)",
"06": {
"emit_tokens": 125,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 1,
"silent_failure": true
},
"07": {
"emit_tokens": 33,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 0,
"silent_failure": false
}
},
{
"scenario": "session-log-flaky-network",
"fault": "one-time 503 (transient)",
"06": {
"emit_tokens": 64,
"claimed_ok": true,
"detected": false,
"persisted": false,
"duplicates": 0,
"silent_failure": true
},
"07": {
"emit_tokens": 17,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 0,
"silent_failure": false
}
},
{
"scenario": "heartbeat-phantom-write",
"fault": "accepted-but-not-persisted",
"06": {
"emit_tokens": 61,
"claimed_ok": true,
"detected": false,
"persisted": false,
"duplicates": 0,
"silent_failure": true
},
"07": {
"emit_tokens": 14,
"claimed_ok": false,
"detected": true,
"persisted": false,
"duplicates": 0,
"silent_failure": false
}
},
{
"scenario": "cold-start-load-6-reads",
"fault": "(none)",
"06": {
"emit_tokens": 192,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 0,
"silent_failure": false
},
"07": {
"emit_tokens": 64,
"claimed_ok": true,
"detected": false,
"persisted": true,
"duplicates": 0,
"silent_failure": false
}
}
],
"totals": {
"0.6": {
"gen_tokens": 608,
"silent_failures": 4,
"detected_failures": 0,
"duplicates": 1,
"effective_tokens": 6608
},
"0.7": {
"gen_tokens": 174,
"silent_failures": 0,
"detected_failures": 2,
"duplicates": 0,
"effective_tokens": 334
"baseline (keyword, entities only)": {
"precision_at_5": 0.2,
"recall_at_5": 0.75,
"mrr": 0.75,
"session_journal_queries_answered": "0/2",
"freshness_top1_correct": false
}
},
"silent_error_free": {
"0.6": "1/5",
"0.7": "5/5"
"dedup": {
"gate ON (v1.5.1 default)": {
"duplicate_notes_created": 0,
"legitimate_captures_blocked": 0,
"referring_attempts": 3,
"legit_attempts": 2
},
"gate OFF (pre-1.5, warn-only)": {
"duplicate_notes_created": 3,
"legitimate_captures_blocked": 0,
"referring_attempts": 3,
"legit_attempts": 2
}
},
"writes_persisted": {
"0.6": "2/5",
"0.7": "3/5"
"write_safety": {
"fault_scenarios": 4,
"silent_write_failures": 0,
"failures_surfaced_loud": 2
},
"durability": {
"writes_during_outage": 3,
"reported_queued": 3,
"landed_after_flush": 3,
"lost": 0,
"duplicated_on_reflush": 0
}
}