jason/chorus
1
0
Fork 0
forked from jason/echo
Code Pull Requests Activity

ver 0.9

Browse Source
This commit is contained in:
jason
2026-06-21 11:46:54 -05:00
parent 88210a4e84
commit d404f6e96f
38 changed files with 2561 additions and 1046 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eval/README.md
+3 -3
View File
@@ -1,7 +1,7 @@
# echo-memory eval — 0.6 vs 0.7 A/B harness
A reproducible, credential-free A/B comparison of the plugin **before** (0.6: raw-curl
recipes from `SKILL.md`) and **after** (0.7: the shipped `scripts/echo.sh` client) the
recipes from `SKILL.md`) and **after** (the shipped `scripts/echo.py` client) the
hardening work. It quantifies the claims in the comparative analysis: token cost of the
I/O layer, and the rate of **silent write failures**.
@@ -14,7 +14,7 @@ python3 run_eval.py --recovery 2500 # sensitivity-test the recovery assumption
python3 run_eval.py --cpt 3.5 # different chars/token proxy
```
No network, no API key, no live vault. Pure stdlib (Python 3 + bash for `echo.sh`).
No network, no API key, no live vault. Pure stdlib Python 3 (the shipped client is now Python — no bash required).
Results table prints to stdout and a machine-readable copy lands in `results/latest.json`.
## How it works
@@ -29,7 +29,7 @@ Results table prints to stdout and a machine-readable copy lands in `results/lat
- a `PATCH` to a missing heading → `400` (the silent-write-loss trigger).
- **`run_eval.py`** — for each scenario, runs both methods against a freshly reset +
re-seeded server (so faults are identical for both), then reads ground truth back
**independently** from the mock. The 0.7 side executes the *actual shipped `echo.sh`*;
**independently** from the mock. The 0.7 side executes the *actual shipped `echo.py`*;
the 0.6 side faithfully models the documented recipe (real HTTP, but no status check,
no retry, no verify, no dedupe).
eval/results/latest.json
+2 -2
View File
@@ -2,7 +2,7 @@
"params": {
"port": 8799,
"cpt": 4.0,
"recovery": 3000,
"recovery": 1500,
"detect_cost": 80
},
"rows": [
@@ -133,7 +133,7 @@
"silent_failures": 4,
"detected_failures": 0,
"duplicates": 1,
"effective_tokens": 12723
"effective_tokens": 6723
},
"0.7": {
"gen_tokens": 174,
eval/run_eval.py
+5 -5
View File
@@ -1,13 +1,13 @@
#!/usr/bin/env python3
"""run_eval.py — A/B eval of echo-memory 0.6 (raw curl, documented recipes) vs
0.7 (the shipped echo.sh client) over representative memory operations, with
0.7 (the shipped echo.py client) over representative memory operations, with
fault injection.
Design
------
* A mock Obsidian REST API (mock_olrapi.py) gives deterministic behavior + faults,
so no credentials are needed and the real vault is never touched.
* The 0.7 side runs the ACTUAL shipped scripts/echo.sh (status-checked, retry, verify,
* The 0.7 side runs the ACTUAL shipped scripts/echo.py (status-checked, retry, verify,
idempotent append).
* The 0.6 side faithfully models the documented raw-curl recipes: it performs the
same HTTP but does NOT inspect status, does NOT retry, does NOT verify, and does
@@ -32,7 +32,7 @@ import os, sys, json, time, subprocess, tempfile, argparse, urllib.request, urll
from pathlib import Path
HERE = Path(__file__).resolve().parent
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.sh"
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
# ----- tiny HTTP helpers (used for setup, ground truth, and the 0.6 model) -----
@@ -59,7 +59,7 @@ class Eval:
def toks(self, text): return round(len(text) / self.cpt)
def echo(self, *args):
env = dict(os.environ, ECHO_BASE=self.base, ECHO_KEY=KEY, ECHO_VERIFY="1", ECHO_LOCK_TTL="900")
p = subprocess.run(["bash", str(ECHO), *args], capture_output=True, text=True, env=env)
p = subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env)
return p.returncode
# ---- faithful 0.6 recipe text (what the model emitted), for token accounting ----
@@ -163,7 +163,7 @@ def scenarios(ev):
http("PUT", f"{ev.base}/vault/{path}", body) # single shot, no retry, status ignored
return {"emit": recipe_put(path), "claimed_ok": True, "detected": False}
def m07():
rc = ev.echo("put", path, tmpfile(body)) # echo.sh retries the 503 once
rc = ev.echo("put", path, tmpfile(body)) # echo.py retries the 503 once
return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0}
def gt():
c = ev.ground(path) or ""
eval/test_features.py
+135
View File
@@ -0,0 +1,135 @@
#!/usr/bin/env python3
"""test_features.py — end-to-end test of the v0.9 features against the mock OLRAPI.
Exercises the real shipped scripts (echo.py capture/resolve/recall/link + sweep.py)
through subprocess, then reads ground truth back from the mock. No network, no creds,
no live vault.
Run: python test_features.py [--port 8801]
"""
import argparse, json, os, subprocess, sys, time, urllib.request, urllib.error
from pathlib import Path
HERE = Path(__file__).resolve().parent
SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"
ECHO = SCRIPTS / "echo.py"
SWEEP = SCRIPTS / "sweep.py"
KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
failures = []
def check(name, cond, detail=""):
print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else ""))
if not cond:
failures.append(name)
class Harness:
def __init__(self, base):
self.base = base
def http(self, method, url, body=None, headers=None):
data = body.encode() if isinstance(body, str) else body
req = urllib.request.Request(url, data=data, method=method,
headers={"Authorization": f"Bearer {KEY}", **(headers or {})})
try:
with urllib.request.urlopen(req, timeout=10) as r:
return r.status, r.read().decode("utf-8", "replace")
except urllib.error.HTTPError as e:
return e.code, e.read().decode("utf-8", "replace")
except Exception as e:
return 0, str(e)
def reset(self):
self.http("POST", f"{self.base}/__debug__reset")
def seed(self, path, content):
self.http("PUT", f"{self.base}/vault/{path}", content)
def ground(self, path):
_, body = self.http("GET", f"{self.base}/__debug__?path={path}")
return None if body == "<<MISSING>>" else body
def echo(self, *args):
env = dict(os.environ, ECHO_BASE=self.base, ECHO_KEY=KEY, ECHO_VERIFY="1",
ECHO_TODAY="2026-06-21")
return subprocess.run([sys.executable, str(args[0]), *args[1:]],
capture_output=True, text=True, env=env)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--port", type=int, default=8801)
a = ap.parse_args()
base = f"http://127.0.0.1:{a.port}"
srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
try:
for _ in range(50):
try:
urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break
except Exception:
time.sleep(0.1)
h = Harness(base)
h.reset()
h.seed("_agent/echo-vault.md", "---\nschema_version: 2\n---\n# marker\n")
# 1. capture a person (with an alias)
r = h.echo(ECHO, "capture", "Bob Smith", "--kind", "person", "--aliases", "bob,rs")
note = h.ground("resources/people/bob-smith.md")
check("capture person creates note", note is not None and "type: person" in note, r.stderr)
check("capture stamps agent_written", note and "agent_written: true" in note)
idx = h.ground("_agent/index/entities.json")
check("index records entity", idx and "bob-smith" in idx and '"bob"' in idx)
# 2. resolve by alias
r = h.echo(ECHO, "resolve", "bob")
out = json.loads(r.stdout) if r.stdout.strip().startswith("{") else {}
check("resolve alias -> path", out.get("match") and out.get("path") == "resources/people/bob-smith.md", r.stdout)
# 3. capture a company whose body mentions Bob Smith -> auto bidirectional link
r = subprocess.run([sys.executable, str(ECHO), "capture", "MPM", "--kind", "company", "-"],
input="Bob Smith is the principal here.\n", capture_output=True, text=True,
env=dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21"))
mpm = h.ground("resources/companies/mpm.md")
bob = h.ground("resources/people/bob-smith.md")
check("auto-link forward (mpm -> bob)", mpm and "[[resources/people/bob-smith]]" in mpm, r.stdout + r.stderr)
check("auto-link reciprocal (bob -> mpm)", bob and "[[resources/companies/mpm]]" in bob)
# 4. recall an entity surfaces its linked neighborhood (search is mocked empty,
# so this exercises the index-resolve + 1-hop expansion path)
r = h.echo(ECHO, "recall", "MPM")
check("recall surfaces linked person", "resources/people/bob-smith.md" in r.stdout, r.stdout)
# 5. explicit link between two fresh notes
h.echo(ECHO, "capture", "Alpha", "--kind", "concept")
h.echo(ECHO, "capture", "Beta", "--kind", "concept")
h.echo(ECHO, "link", "resources/concepts/alpha.md", "resources/concepts/beta.md")
alpha = h.ground("resources/concepts/alpha.md")
beta = h.ground("resources/concepts/beta.md")
check("link adds A->B", alpha and "[[resources/concepts/beta]]" in alpha)
check("link adds B->A", beta and "[[resources/concepts/alpha]]" in beta)
# 6. inbox capture (unknown home)
h.echo(ECHO, "capture", "stray thought", "--inbox")
inbox = h.ground("inbox/captures/inbox.md")
check("inbox capture lands", inbox and "stray thought" in inbox)
# 7. sweep --apply rebuilds the index, symmetrizes, stamps schema 3
r = h.echo(SWEEP, "--apply")
marker = h.ground("_agent/echo-vault.md")
check("sweep runs clean", r.returncode == 0, r.stdout + r.stderr)
check("sweep stamps schema 3", marker and "schema_version=3" in marker.replace(": ", "="), marker)
idx2 = h.ground("_agent/index/entities.json")
check("sweep rebuilt index has all entities",
idx2 and all(s in idx2 for s in ["bob-smith", "mpm", "alpha", "beta"]))
print(f"\n{len(failures)} failure(s)" if failures else "\nall feature tests passed")
return 1 if failures else 0
finally:
srv.terminate()
if __name__ == "__main__":
raise SystemExit(main())