echo/echo-memory.plugin.src/skills/echo-memory/eval/perf/fixtures.py

#!/usr/bin/env python3
"""fixtures.py — inputs for the ECHO performance harness (bench.py).

Holds the *what* (subjects to pull, notes to write, mentions to resolve) so the
harness (bench.py) stays the *how* (timing, stats, gating). Nothing here touches
the network; bench.py owns all I/O.
"""

from __future__ import annotations

# --- §3.2 subject pulls -------------------------------------------------------
# (label, query) pairs fed to `recall`. Chosen to span neighbourhood sizes — the
# real cost driver — from a likely-leaf to a known hub. APTA and CapMetro are the
# subjects the maintainer configured; "echo" and "operator preferences" are dense hubs
# that stress the 1-hop expansion (the read_many fan-out 1.1.0 accelerated).
SUBJECTS: list[tuple[str, str]] = [
    ("apta", "APTA"),
    ("capmetro", "CapMetro"),
    ("hub-echo", "echo"),
    ("hub-prefs", "operator preferences"),
    ("leaf-rivnut", "rivnut torque spec"),
]


# --- §4.3 fuzzy-resolve correctness table (1.2.0) -----------------------------
# Each case: mention, expected behaviour. `expect_slug` is the canonical slug an
# exact/alias/fuzzy match should land on (None = no entity expected). `expect`
# is the classifier:
#   "exact"      -> resolve returns an entity whose slug == expect_slug
#   "candidates" -> no exact hit, but fuzzy_candidates surfaces expect_slug
#                   (the 1.2.0 anti-duplicate guard — a near-miss must NOT resolve
#                    to nothing and silently spawn a new note)
#   "miss"       -> neither; genuinely unknown
# Tune expect_slug to the live vault; unknowns are reported as INFO, not failures.
# NOTE: the subject/slug values below are neutral placeholders. The maintainer
# should point these fixtures at dense hubs that actually exist in their own
# vault, otherwise the resolve cases will report INFO misses rather than hits.
RESOLVE_CASES: list[dict] = [
    {"mention": "echo", "expect": "exact", "expect_slug": "echo"},
    {"mention": "echo memory", "expect": "candidates", "expect_slug": "echo"},
    {"mention": "ECHO plugin", "expect": "candidates", "expect_slug": "echo"},
    {"mention": "other-vault", "expect": "exact", "expect_slug": "other-vault"},
    {"mention": "example subject", "expect": "exact", "expect_slug": "example-subject"},
    {"mention": "operator", "expect": "candidates", "expect_slug": "example-subject"},
    {"mention": "zzqx nonexistent entity", "expect": "miss", "expect_slug": None},
]


# --- §3.3 bulk write fixtures -------------------------------------------------
BENCH_PREFIX = "_agent/_bench"  # run-scoped namespace: _agent/_bench/<run-id>/...


def note_body(run_id: str, i: int) -> str:
    """A realistic-shape note: canonical frontmatter + a couple of headings, so
    PUT cost reflects a true note, not a one-liner. agent_written + a _bench tag
    make these trivially greppable if a cleanup is ever missed."""
    return (
        "---\n"
        "type: working-memory\n"
        "status: active\n"
        "created: 2026-06-23\n"
        "updated: 2026-06-23\n"
        "tags:\n"
        "  - agent\n"
        "  - _bench\n"
        "agent_written: true\n"
        f"source_notes: []\n"
        f"bench_run: {run_id}\n"
        "---\n\n"
        f"# Bench Note {i:04d}\n\n"
        "## Body\n"
        f"Synthetic benchmark note {i} for run {run_id}. "
        "Filler so the payload is not pathologically small: "
        + ("lorem ipsum dolor sit amet " * 6)
        + "\n\n## Log\n- seed\n"
    )


def note_path(run_id: str, i: int) -> str:
    return f"{BENCH_PREFIX}/{run_id}/note-{i:04d}.md"


def append_target(run_id: str) -> str:
    return f"{BENCH_PREFIX}/{run_id}/append-target.md"


def append_target_seed(run_id: str) -> str:
    return (
        "---\n"
        "type: working-memory\n"
        "status: active\n"
        "created: 2026-06-23\n"
        "updated: 2026-06-23\n"
        "tags:\n"
        "  - agent\n"
        "  - _bench\n"
        "agent_written: true\n"
        "source_notes: []\n"
        f"bench_run: {run_id}\n"
        "---\n\n"
        "# Append Target\n\n"
        "## Log\n"
    )