testing and documentation

2026-06-23 17:17:00 -05:00
parent c2da2d261c
commit e89f4660f6
28 changed files with 4045 additions and 19 deletions
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""fixtures.py — inputs for the ECHO performance harness (bench.py).
+
+Holds the *what* (subjects to pull, notes to write, mentions to resolve) so the
+harness (bench.py) stays the *how* (timing, stats, gating). Nothing here touches
+the network; bench.py owns all I/O.
+"""
+
+from __future__ import annotations
+
+# --- §3.2 subject pulls -------------------------------------------------------
+# (label, query) pairs fed to `recall`. Chosen to span neighbourhood sizes — the
+# real cost driver — from a likely-leaf to a known hub. APTA and CapMetro are the
+# subjects Jason named; "echo" and "operator preferences" are dense hubs that
+# stress the 1-hop expansion (the read_many fan-out 1.1.0 accelerated).
+SUBJECTS: list[tuple[str, str]] = [
+    ("apta", "APTA"),
+    ("capmetro", "CapMetro"),
+    ("hub-echo", "echo"),
+    ("hub-prefs", "operator preferences"),
+    ("leaf-rivnut", "rivnut torque spec"),
+]
+
+
+# --- §4.3 fuzzy-resolve correctness table (1.2.0) -----------------------------
+# Each case: mention, expected behaviour. `expect_slug` is the canonical slug an
+# exact/alias/fuzzy match should land on (None = no entity expected). `expect`
+# is the classifier:
+#   "exact"      -> resolve returns an entity whose slug == expect_slug
+#   "candidates" -> no exact hit, but fuzzy_candidates surfaces expect_slug
+#                   (the 1.2.0 anti-duplicate guard — a near-miss must NOT resolve
+#                    to nothing and silently spawn a new note)
+#   "miss"       -> neither; genuinely unknown
+# Tune expect_slug to the live vault; unknowns are reported as INFO, not failures.
+RESOLVE_CASES: list[dict] = [
+    {"mention": "echo", "expect": "exact", "expect_slug": "echo"},
+    {"mention": "echo memory", "expect": "candidates", "expect_slug": "echo"},
+    {"mention": "ECHO plugin", "expect": "candidates", "expect_slug": "echo"},
+    {"mention": "goldbrain", "expect": "exact", "expect_slug": "goldbrain"},
+    {"mention": "jason stedwell", "expect": "exact", "expect_slug": "jason-stedwell"},
+    {"mention": "jason", "expect": "candidates", "expect_slug": "jason-stedwell"},
+    {"mention": "zzqx nonexistent entity", "expect": "miss", "expect_slug": None},
+]
+
+
+# --- §3.3 bulk write fixtures -------------------------------------------------
+BENCH_PREFIX = "_agent/_bench"  # run-scoped namespace: _agent/_bench/<run-id>/...
+
+
+def note_body(run_id: str, i: int) -> str:
+    """A realistic-shape note: canonical frontmatter + a couple of headings, so
+    PUT cost reflects a true note, not a one-liner. agent_written + a _bench tag
+    make these trivially greppable if a cleanup is ever missed."""
+    return (
+        "---\n"
+        "type: working-memory\n"
+        "status: active\n"
+        "created: 2026-06-23\n"
+        "updated: 2026-06-23\n"
+        "tags:\n"
+        "  - agent\n"
+        "  - _bench\n"
+        "agent_written: true\n"
+        f"source_notes: []\n"
+        f"bench_run: {run_id}\n"
+        "---\n\n"
+        f"# Bench Note {i:04d}\n\n"
+        "## Body\n"
+        f"Synthetic benchmark note {i} for run {run_id}. "
+        "Filler so the payload is not pathologically small: "
+        + ("lorem ipsum dolor sit amet " * 6)
+        + "\n\n## Log\n- seed\n"
+    )
+
+
+def note_path(run_id: str, i: int) -> str:
+    return f"{BENCH_PREFIX}/{run_id}/note-{i:04d}.md"
+
+
+def append_target(run_id: str) -> str:
+    return f"{BENCH_PREFIX}/{run_id}/append-target.md"
+
+
+def append_target_seed(run_id: str) -> str:
+    return (
+        "---\n"
+        "type: working-memory\n"
+        "status: active\n"
+        "created: 2026-06-23\n"
+        "updated: 2026-06-23\n"
+        "tags:\n"
+        "  - agent\n"
+        "  - _bench\n"
+        "agent_written: true\n"
+        "source_notes: []\n"
+        f"bench_run: {run_id}\n"
+        "---\n\n"
+        "# Append Target\n\n"
+        "## Log\n"
+    )