forked from jason/echo
Retire ROADMAP-1.0; rewrite run_eval.py as current-version metrics harness
Knocks two items off ROADMAP-2.0 (§4 eval refresh, §5 roadmap retirement): - run_eval.py: the stale 0.6-vs-0.7 A/B is replaced (old version in git history) with a four-part current-version eval against the mock — retrieval (hybrid+priors vs keyword/entities-only baseline: R@5 1.00 vs 0.75, MRR 1.00 vs 0.75, session/journal queries 2/2 vs 0/2, freshness top-1 correct vs wrong), dedup (0 dupes gate-on vs 3 gate-off, 0 false blocks), write safety (0 silent failures across 4 fault scenarios), durability (3/3 offline writes queued+landed, 0 lost, 0 re-flush dupes). Results in eval/results/latest.json. - README: metrics table published; repo-layout eval descriptor updated. - eval/README: new harness documented; A/B framing marked historical. - ROADMAP-1.0.md removed (fully shipped; story lives in the README version table + git history); dangling docstring pointer fixed in echo_concurrency.py; ROADMAP-2.0 checkboxes ticked. All suites green; artifact rebuilt (1.5.1, docstring-only source change). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+39
-147
@@ -1,154 +1,46 @@
|
||||
{
|
||||
"params": {
|
||||
"port": 8799,
|
||||
"cpt": 4.0,
|
||||
"recovery": 1500,
|
||||
"detect_cost": 80
|
||||
},
|
||||
"rows": [
|
||||
{
|
||||
"scenario": "agent-log-missing-heading",
|
||||
"fault": "bad heading -> 400 invalid-target",
|
||||
"06": {
|
||||
"emit_tokens": 82,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 22,
|
||||
"claimed_ok": false,
|
||||
"detected": true,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
"version": "1.5.1",
|
||||
"date": "2026-07-03",
|
||||
"retrieval": {
|
||||
"current (hybrid+priors, full corpus)": {
|
||||
"precision_at_5": 0.25,
|
||||
"recall_at_5": 1.0,
|
||||
"mrr": 1.0,
|
||||
"session_journal_queries_answered": "2/2",
|
||||
"freshness_top1_correct": true
|
||||
},
|
||||
{
|
||||
"scenario": "scope-switch",
|
||||
"fault": "(none)",
|
||||
"06": {
|
||||
"emit_tokens": 84,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 24,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "inbox-capture-replayed",
|
||||
"fault": "duplicate attempt (retry/replay)",
|
||||
"06": {
|
||||
"emit_tokens": 125,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 1,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 33,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "session-log-flaky-network",
|
||||
"fault": "one-time 503 (transient)",
|
||||
"06": {
|
||||
"emit_tokens": 64,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 17,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "heartbeat-phantom-write",
|
||||
"fault": "accepted-but-not-persisted",
|
||||
"06": {
|
||||
"emit_tokens": 61,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": true
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 14,
|
||||
"claimed_ok": false,
|
||||
"detected": true,
|
||||
"persisted": false,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"scenario": "cold-start-load-6-reads",
|
||||
"fault": "(none)",
|
||||
"06": {
|
||||
"emit_tokens": 192,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
},
|
||||
"07": {
|
||||
"emit_tokens": 64,
|
||||
"claimed_ok": true,
|
||||
"detected": false,
|
||||
"persisted": true,
|
||||
"duplicates": 0,
|
||||
"silent_failure": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"totals": {
|
||||
"0.6": {
|
||||
"gen_tokens": 608,
|
||||
"silent_failures": 4,
|
||||
"detected_failures": 0,
|
||||
"duplicates": 1,
|
||||
"effective_tokens": 6608
|
||||
},
|
||||
"0.7": {
|
||||
"gen_tokens": 174,
|
||||
"silent_failures": 0,
|
||||
"detected_failures": 2,
|
||||
"duplicates": 0,
|
||||
"effective_tokens": 334
|
||||
"baseline (keyword, entities only)": {
|
||||
"precision_at_5": 0.2,
|
||||
"recall_at_5": 0.75,
|
||||
"mrr": 0.75,
|
||||
"session_journal_queries_answered": "0/2",
|
||||
"freshness_top1_correct": false
|
||||
}
|
||||
},
|
||||
"silent_error_free": {
|
||||
"0.6": "1/5",
|
||||
"0.7": "5/5"
|
||||
"dedup": {
|
||||
"gate ON (v1.5.1 default)": {
|
||||
"duplicate_notes_created": 0,
|
||||
"legitimate_captures_blocked": 0,
|
||||
"referring_attempts": 3,
|
||||
"legit_attempts": 2
|
||||
},
|
||||
"gate OFF (pre-1.5, warn-only)": {
|
||||
"duplicate_notes_created": 3,
|
||||
"legitimate_captures_blocked": 0,
|
||||
"referring_attempts": 3,
|
||||
"legit_attempts": 2
|
||||
}
|
||||
},
|
||||
"writes_persisted": {
|
||||
"0.6": "2/5",
|
||||
"0.7": "3/5"
|
||||
"write_safety": {
|
||||
"fault_scenarios": 4,
|
||||
"silent_write_failures": 0,
|
||||
"failures_surfaced_loud": 2
|
||||
},
|
||||
"durability": {
|
||||
"writes_during_outage": 3,
|
||||
"reported_queued": 3,
|
||||
"landed_after_flush": 3,
|
||||
"lost": 0,
|
||||
"duplicated_on_reflush": 0
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user