forked from jason/echo
Phase 1: mechanical echo→chorus rename with back-compat shims
Identity rename, no behavior change (CHORUS-PLAN.md Phase 1): - Plugin echo-memory → chorus-memory: manifest (v2.0.0-alpha.1), skill dir, 16 scripts (chorus.py, chorus_config.py, …), EchoError → ChorusError, /chorus-* commands, hook paths, docs, scaffold seeds, eval harness, build.py. Docs endpoint → chorusapi.mpm.to. - Env ECHO_* → CHORUS_*; config → ~/.claude/chorus-memory/config.json; state dir → ~/.chorus-memory/; marker → _agent/chorus-vault.md. Back-compat shims (one major version): - chorus_config aliases legacy ECHO_* env at import; reads a legacy echo-memory config.json when no CHORUS config exists (writes never land there); doctor/config report the legacy source. - State dir honors an existing ~/.echo-memory when the new dir is absent (offline queue not stranded). - Marker dual-probe in load/doctor/bootstrap/lint/sweep/migrate: a pre-fork _agent/echo-vault.md counts as bootstrapped; bootstrap repair won't write a second marker; routing gains agent-marker-legacy (GET). Verified: 25/25 unit tests, scaffold + routing-sync checks, 4 mock e2e suites, run_eval metrics unchanged, +6 shim smoke tests green. Rebuilt chorus-memory.plugin (79 entries). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+29
-29
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
"""run_eval.py — current-version metrics for echo-memory (v1.5+).
|
||||
"""run_eval.py — current-version metrics for chorus-memory (v1.5+).
|
||||
|
||||
Replaces the historical 0.6-vs-0.7 A/B harness (see git history for that version).
|
||||
Measures the four things the plugin now claims, against the deterministic mock
|
||||
@@ -11,7 +11,7 @@ REST API (mock_olrapi.py) — no credentials, no live vault:
|
||||
Metrics: precision@5, recall@5, MRR, session/journal queries
|
||||
answered, freshness top-1 correctness.
|
||||
2. DEDUP — duplicate notes created replaying a capture stream with the
|
||||
pre-write gate ON (default) vs OFF (ECHO_DUP_GATE=99, i.e. the
|
||||
pre-write gate ON (default) vs OFF (CHORUS_DUP_GATE=99, i.e. the
|
||||
pre-1.5 warn-only behavior). Also counts legitimate captures
|
||||
wrongly blocked (must be 0).
|
||||
3. WRITE SAFETY— silent write failures (claimed success but ground truth wrong)
|
||||
@@ -40,8 +40,8 @@ import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"
|
||||
ECHO = SCRIPTS / "echo.py"
|
||||
SCRIPTS = HERE.parent / "chorus-memory.plugin.src" / "skills" / "chorus-memory" / "scripts"
|
||||
CHORUS = SCRIPTS / "chorus.py"
|
||||
SWEEP = SCRIPTS / "sweep.py"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
TODAY = "2026-07-03"
|
||||
@@ -76,8 +76,8 @@ class Harness:
|
||||
return None if body == "<<MISSING>>" else body
|
||||
|
||||
def run(self, script, *args, env_extra=None, base=None):
|
||||
env = dict(os.environ, ECHO_BASE=base or self.base, ECHO_KEY=KEY,
|
||||
ECHO_VERIFY="1", ECHO_TODAY=TODAY, ECHO_STATE_DIR=self.state_dir,
|
||||
env = dict(os.environ, CHORUS_BASE=base or self.base, CHORUS_KEY=KEY,
|
||||
CHORUS_VERIFY="1", CHORUS_TODAY=TODAY, CHORUS_STATE_DIR=self.state_dir,
|
||||
**(env_extra or {}))
|
||||
return subprocess.run([sys.executable, str(script), *args],
|
||||
capture_output=True, text=True, env=env)
|
||||
@@ -179,7 +179,7 @@ def metrics_for(ranked_by_query):
|
||||
|
||||
def eval_retrieval(h):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
for path, text in {**ENTITY_CORPUS, **TIME_SERIES_CORPUS}.items():
|
||||
h.seed(path, text)
|
||||
r = h.run(SWEEP, "--apply")
|
||||
@@ -188,7 +188,7 @@ def eval_retrieval(h):
|
||||
current_ranked, baseline_ranked = [], []
|
||||
fresh_top1_current = fresh_top1_baseline = None
|
||||
for q, gold, ts_only in QUERIES:
|
||||
r = h.run(ECHO, "recall", q, "--json", "--limit", "5")
|
||||
r = h.run(CHORUS, "recall", q, "--json", "--limit", "5")
|
||||
try:
|
||||
primary = [hit["path"] for hit in json.loads(r.stdout).get("primary", [])]
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -212,28 +212,28 @@ def eval_retrieval(h):
|
||||
# Attempts referring to an EXISTING entity under another name; a create == duplicate.
|
||||
DUP_ATTEMPTS = [
|
||||
("Robert Smith", "person", "resources/people/robert-smith.md"),
|
||||
("Echo Memory", "project", "projects/active/echo-memory.md"),
|
||||
("Chorus Memory", "project", "projects/active/chorus-memory.md"),
|
||||
("Acme Consulting", "company", "resources/companies/acme-consulting.md"),
|
||||
]
|
||||
# Legitimate NEW entities; a block == false positive (must create in both modes).
|
||||
LEGIT_ATTEMPTS = [
|
||||
("Echo Review Meeting", "meeting",
|
||||
f"resources/meetings/{TODAY}-echo-review-meeting.md"), # cross-kind lookalike
|
||||
("Chorus Review Meeting", "meeting",
|
||||
f"resources/meetings/{TODAY}-chorus-review-meeting.md"), # cross-kind lookalike
|
||||
("Zephyr", "concept", "resources/concepts/zephyr.md"), # unrelated
|
||||
]
|
||||
|
||||
|
||||
def _dedup_pass(h, gate_env):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
for title, kind in [("Bob Smith", "person"), ("Echo", "project"), ("Acme", "company")]:
|
||||
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
for title, kind in [("Bob Smith", "person"), ("Chorus", "project"), ("Acme", "company")]:
|
||||
h.run(CHORUS, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
dupes = blocked_legit = 0
|
||||
for title, kind, would_be in DUP_ATTEMPTS:
|
||||
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
h.run(CHORUS, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
dupes += int(h.ground(would_be) is not None)
|
||||
for title, kind, path in LEGIT_ATTEMPTS:
|
||||
h.run(ECHO, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
h.run(CHORUS, "capture", title, "--kind", kind, env_extra=gate_env)
|
||||
blocked_legit += int(h.ground(path) is None)
|
||||
return {"duplicate_notes_created": dupes, "legitimate_captures_blocked": blocked_legit,
|
||||
"referring_attempts": len(DUP_ATTEMPTS), "legit_attempts": len(LEGIT_ATTEMPTS)}
|
||||
@@ -241,13 +241,13 @@ def _dedup_pass(h, gate_env):
|
||||
|
||||
def eval_dedup(h):
|
||||
return {"gate ON (v1.5.1 default)": _dedup_pass(h, {}),
|
||||
"gate OFF (pre-1.5, warn-only)": _dedup_pass(h, {"ECHO_DUP_GATE": "99"})}
|
||||
"gate OFF (pre-1.5, warn-only)": _dedup_pass(h, {"CHORUS_DUP_GATE": "99"})}
|
||||
|
||||
|
||||
# ------------------------------------------------------------- 3. WRITE SAFETY
|
||||
def eval_write_safety(h):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
silent = loud = 0
|
||||
ops = 0
|
||||
|
||||
@@ -259,7 +259,7 @@ def eval_write_safety(h):
|
||||
|
||||
# transient 503 on first write ("flaky" marker) -> client must retry and land it
|
||||
ops += 1
|
||||
r = h.run(ECHO, "put", "resources/concepts/flaky-note.md", bodyfile("# Flaky\nok\n"))
|
||||
r = h.run(CHORUS, "put", "resources/concepts/flaky-note.md", bodyfile("# Flaky\nok\n"))
|
||||
landed = h.ground("resources/concepts/flaky-note.md") is not None
|
||||
if r.returncode == 0 and not landed:
|
||||
silent += 1
|
||||
@@ -268,7 +268,7 @@ def eval_write_safety(h):
|
||||
|
||||
# phantom PUT ("phantom" marker: 200 but not persisted) -> verify must fail LOUD
|
||||
ops += 1
|
||||
r = h.run(ECHO, "put", "resources/concepts/phantom-note.md", bodyfile("# Phantom\n"))
|
||||
r = h.run(CHORUS, "put", "resources/concepts/phantom-note.md", bodyfile("# Phantom\n"))
|
||||
landed = h.ground("resources/concepts/phantom-note.md") is not None
|
||||
if r.returncode == 0 and not landed:
|
||||
silent += 1
|
||||
@@ -278,7 +278,7 @@ def eval_write_safety(h):
|
||||
# PATCH to a missing heading -> 400 must be surfaced, nothing written
|
||||
ops += 1
|
||||
h.seed("resources/concepts/target.md", "# Target\n\n## Real\nx\n")
|
||||
r = h.run(ECHO, "patch", "resources/concepts/target.md", "append", "heading",
|
||||
r = h.run(CHORUS, "patch", "resources/concepts/target.md", "append", "heading",
|
||||
"Target::Nope", bodyfile("lost?"))
|
||||
if r.returncode == 0 and "lost?" not in (h.ground("resources/concepts/target.md") or ""):
|
||||
silent += 1
|
||||
@@ -287,8 +287,8 @@ def eval_write_safety(h):
|
||||
|
||||
# replayed append -> whole-line idempotency, no duplicate
|
||||
ops += 1
|
||||
h.run(ECHO, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
||||
h.run(ECHO, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
||||
h.run(CHORUS, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
||||
h.run(CHORUS, "append", "inbox/captures/inbox.md", f"- {TODAY}: replay me")
|
||||
n = (h.ground("inbox/captures/inbox.md") or "").count("replay me")
|
||||
if n != 1:
|
||||
silent += 1
|
||||
@@ -300,15 +300,15 @@ def eval_write_safety(h):
|
||||
# --------------------------------------------------------------- 4. DURABILITY
|
||||
def eval_durability(h, dead_base):
|
||||
h.reset()
|
||||
h.seed("_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
h.seed("_agent/chorus-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
queued_ok = 0
|
||||
for i in range(3):
|
||||
r = h.run(ECHO, "append", "inbox/captures/inbox.md",
|
||||
r = h.run(CHORUS, "append", "inbox/captures/inbox.md",
|
||||
f"- {TODAY}: offline capture {i}", base=dead_base)
|
||||
queued_ok += int(r.returncode == 0 and "queued" in r.stdout)
|
||||
# vault returns: flush replays; a second flush must not duplicate
|
||||
h.run(ECHO, "flush")
|
||||
h.run(ECHO, "flush")
|
||||
h.run(CHORUS, "flush")
|
||||
h.run(CHORUS, "flush")
|
||||
inbox = h.ground("inbox/captures/inbox.md") or ""
|
||||
landed = sum(1 for i in range(3) if f"offline capture {i}" in inbox)
|
||||
dupes = sum(inbox.count(f"offline capture {i}") - 1
|
||||
@@ -325,7 +325,7 @@ def main():
|
||||
base = f"http://127.0.0.1:{a.port}"
|
||||
srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
state = tempfile.mkdtemp(prefix="echo-eval-state-")
|
||||
state = tempfile.mkdtemp(prefix="chorus-eval-state-")
|
||||
try:
|
||||
for _ in range(50):
|
||||
try:
|
||||
@@ -345,7 +345,7 @@ def main():
|
||||
out.parent.mkdir(exist_ok=True)
|
||||
out.write_text(json.dumps(results, indent=2), encoding="utf-8")
|
||||
|
||||
print(f"\n== echo-memory eval — v{results['version']} ({results['date']}) ==\n")
|
||||
print(f"\n== chorus-memory eval — v{results['version']} ({results['date']}) ==\n")
|
||||
for section, data in results.items():
|
||||
if section in ("version", "date"):
|
||||
continue
|
||||
|
||||
Reference in New Issue
Block a user