forked from jason/echo
Jason Stedwell
993abdc846
Knocks two items off ROADMAP-2.0 (§4 eval refresh, §5 roadmap retirement): - run_eval.py: the stale 0.6-vs-0.7 A/B is replaced (old version in git history) with a four-part current-version eval against the mock — retrieval (hybrid+priors vs keyword/entities-only baseline: R@5 1.00 vs 0.75, MRR 1.00 vs 0.75, session/journal queries 2/2 vs 0/2, freshness top-1 correct vs wrong), dedup (0 dupes gate-on vs 3 gate-off, 0 false blocks), write safety (0 silent failures across 4 fault scenarios), durability (3/3 offline writes queued+landed, 0 lost, 0 re-flush dupes). Results in eval/results/latest.json. - README: metrics table published; repo-layout eval descriptor updated. - eval/README: new harness documented; A/B framing marked historical. - ROADMAP-1.0.md removed (fully shipped; story lives in the README version table + git history); dangling docstring pointer fixed in echo_concurrency.py; ROADMAP-2.0 checkboxes ticked. All suites green; artifact rebuilt (1.5.1, docstring-only source change). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
46 lines
1.1 KiB
JSON
46 lines
1.1 KiB
JSON
{
|
|
"version": "1.5.1",
|
|
"date": "2026-07-03",
|
|
"retrieval": {
|
|
"current (hybrid+priors, full corpus)": {
|
|
"precision_at_5": 0.25,
|
|
"recall_at_5": 1.0,
|
|
"mrr": 1.0,
|
|
"session_journal_queries_answered": "2/2",
|
|
"freshness_top1_correct": true
|
|
},
|
|
"baseline (keyword, entities only)": {
|
|
"precision_at_5": 0.2,
|
|
"recall_at_5": 0.75,
|
|
"mrr": 0.75,
|
|
"session_journal_queries_answered": "0/2",
|
|
"freshness_top1_correct": false
|
|
}
|
|
},
|
|
"dedup": {
|
|
"gate ON (v1.5.1 default)": {
|
|
"duplicate_notes_created": 0,
|
|
"legitimate_captures_blocked": 0,
|
|
"referring_attempts": 3,
|
|
"legit_attempts": 2
|
|
},
|
|
"gate OFF (pre-1.5, warn-only)": {
|
|
"duplicate_notes_created": 3,
|
|
"legitimate_captures_blocked": 0,
|
|
"referring_attempts": 3,
|
|
"legit_attempts": 2
|
|
}
|
|
},
|
|
"write_safety": {
|
|
"fault_scenarios": 4,
|
|
"silent_write_failures": 0,
|
|
"failures_surfaced_loud": 2
|
|
},
|
|
"durability": {
|
|
"writes_during_outage": 3,
|
|
"reported_queued": 3,
|
|
"landed_after_flush": 3,
|
|
"lost": 0,
|
|
"duplicated_on_reflush": 0
|
|
}
|
|
} |