forked from jason/echo
Jason Stedwell
101 lines
3.8 KiB
Python
101 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""fixtures.py — inputs for the ECHO performance harness (bench.py).
|
|
|
|
Holds the *what* (subjects to pull, notes to write, mentions to resolve) so the
|
|
harness (bench.py) stays the *how* (timing, stats, gating). Nothing here touches
|
|
the network; bench.py owns all I/O.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
# --- §3.2 subject pulls -------------------------------------------------------
|
|
# (label, query) pairs fed to `recall`. Chosen to span neighbourhood sizes — the
|
|
# real cost driver — from a likely-leaf to a known hub. APTA and CapMetro are the
|
|
# subjects Jason named; "echo" and "operator preferences" are dense hubs that
|
|
# stress the 1-hop expansion (the read_many fan-out 1.1.0 accelerated).
|
|
SUBJECTS: list[tuple[str, str]] = [
|
|
("apta", "APTA"),
|
|
("capmetro", "CapMetro"),
|
|
("hub-echo", "echo"),
|
|
("hub-prefs", "operator preferences"),
|
|
("leaf-rivnut", "rivnut torque spec"),
|
|
]
|
|
|
|
|
|
# --- §4.3 fuzzy-resolve correctness table (1.2.0) -----------------------------
|
|
# Each case: mention, expected behaviour. `expect_slug` is the canonical slug an
|
|
# exact/alias/fuzzy match should land on (None = no entity expected). `expect`
|
|
# is the classifier:
|
|
# "exact" -> resolve returns an entity whose slug == expect_slug
|
|
# "candidates" -> no exact hit, but fuzzy_candidates surfaces expect_slug
|
|
# (the 1.2.0 anti-duplicate guard — a near-miss must NOT resolve
|
|
# to nothing and silently spawn a new note)
|
|
# "miss" -> neither; genuinely unknown
|
|
# Tune expect_slug to the live vault; unknowns are reported as INFO, not failures.
|
|
RESOLVE_CASES: list[dict] = [
|
|
{"mention": "echo", "expect": "exact", "expect_slug": "echo"},
|
|
{"mention": "echo memory", "expect": "candidates", "expect_slug": "echo"},
|
|
{"mention": "ECHO plugin", "expect": "candidates", "expect_slug": "echo"},
|
|
{"mention": "goldbrain", "expect": "exact", "expect_slug": "goldbrain"},
|
|
{"mention": "jason stedwell", "expect": "exact", "expect_slug": "jason-stedwell"},
|
|
{"mention": "jason", "expect": "candidates", "expect_slug": "jason-stedwell"},
|
|
{"mention": "zzqx nonexistent entity", "expect": "miss", "expect_slug": None},
|
|
]
|
|
|
|
|
|
# --- §3.3 bulk write fixtures -------------------------------------------------
|
|
BENCH_PREFIX = "_agent/_bench" # run-scoped namespace: _agent/_bench/<run-id>/...
|
|
|
|
|
|
def note_body(run_id: str, i: int) -> str:
|
|
"""A realistic-shape note: canonical frontmatter + a couple of headings, so
|
|
PUT cost reflects a true note, not a one-liner. agent_written + a _bench tag
|
|
make these trivially greppable if a cleanup is ever missed."""
|
|
return (
|
|
"---\n"
|
|
"type: working-memory\n"
|
|
"status: active\n"
|
|
"created: 2026-06-23\n"
|
|
"updated: 2026-06-23\n"
|
|
"tags:\n"
|
|
" - agent\n"
|
|
" - _bench\n"
|
|
"agent_written: true\n"
|
|
f"source_notes: []\n"
|
|
f"bench_run: {run_id}\n"
|
|
"---\n\n"
|
|
f"# Bench Note {i:04d}\n\n"
|
|
"## Body\n"
|
|
f"Synthetic benchmark note {i} for run {run_id}. "
|
|
"Filler so the payload is not pathologically small: "
|
|
+ ("lorem ipsum dolor sit amet " * 6)
|
|
+ "\n\n## Log\n- seed\n"
|
|
)
|
|
|
|
|
|
def note_path(run_id: str, i: int) -> str:
|
|
return f"{BENCH_PREFIX}/{run_id}/note-{i:04d}.md"
|
|
|
|
|
|
def append_target(run_id: str) -> str:
|
|
return f"{BENCH_PREFIX}/{run_id}/append-target.md"
|
|
|
|
|
|
def append_target_seed(run_id: str) -> str:
|
|
return (
|
|
"---\n"
|
|
"type: working-memory\n"
|
|
"status: active\n"
|
|
"created: 2026-06-23\n"
|
|
"updated: 2026-06-23\n"
|
|
"tags:\n"
|
|
" - agent\n"
|
|
" - _bench\n"
|
|
"agent_written: true\n"
|
|
"source_notes: []\n"
|
|
f"bench_run: {run_id}\n"
|
|
"---\n\n"
|
|
"# Append Target\n\n"
|
|
"## Log\n"
|
|
)
|