forked from jason/echo
ver 1.5.0 — six-improvement pass: retention, routing, self-firing memory
1. capture-update keeps the whole body (was truncating to first line) 2. frontmatter completeness: kind-default status + kind-seeded tags at capture, fm create-or-replace, incomplete-frontmatter lint, sweep backfill (one data source: KIND_STATUS/KIND_REQUIRED_FM) 3. pre-write duplicate gate (exit 76, --merge-into/--force) 4. recall corpus + ranking: sessions/journal indexed (down-weighted), BM25 x freshness x status fusion, recall --json, index schema 2 5. session hooks: SessionStart auto-load, Stop reflection nudge 6. one-tap triage verb with processing-log audit; --json on read verbs +22 mock end-to-end tests; all suites green. Docs current (README, CHANGELOG, SKILL.md, commands, references, MAINTENANCE, eval README). Drops tracked .DS_Store (gitignored); retires README-gretchen.md (moved to gitignored dist/); adds the field report that drove item 2. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jason Stedwell
+14
-1
@@ -5,7 +5,20 @@ recipes from `SKILL.md`) and **after** (the shipped `scripts/echo.py` client) th
|
||||
hardening work. It quantifies the claims in the comparative analysis: token cost of the
|
||||
I/O layer, and the rate of **silent write failures**.
|
||||
|
||||
## Run it
|
||||
## Current test suites (run these for any change)
|
||||
|
||||
The A/B harness below is the historical 0.6-vs-0.7 comparison. The **current-version**
|
||||
suites live alongside it and are what CI gates on:
|
||||
|
||||
```bash
|
||||
python3 test_features.py # end-to-end features vs the mock (capture/recall/gate/triage/hooks/…)
|
||||
python3 test_reflect.py # reflection pipeline
|
||||
python3 test_offline_queue.py # H2 outage queue + read cache
|
||||
python3 test_patch_semantics.py # real PATCH semantics vs mock_olrapi_hifi.py (incl. fm create-or-replace)
|
||||
# plus, in the plugin tree: scripts/test_echo_client.py (offline unit + routing-sync guard)
|
||||
```
|
||||
|
||||
## Run it (historical A/B)
|
||||
|
||||
```bash
|
||||
cd eval
|
||||
|
||||
@@ -170,6 +170,158 @@ def main():
|
||||
and env.get("path") == "resources/companies/json-co.md", r.stdout)
|
||||
check("M4 --json capture actually wrote the note", h.ground("resources/companies/json-co.md") is not None)
|
||||
|
||||
# ---------------- v1.5 features ----------------------------------------
|
||||
# step 8 clobbered entities.json on purpose; rebuild the index from the notes
|
||||
# so the v1.5 tests run against a coherent vault.
|
||||
h.echo(SWEEP, "--apply")
|
||||
|
||||
# 10. frontmatter completeness at write time: entity notes are born classified.
|
||||
bob2 = h.ground("resources/people/bob-smith.md") or ""
|
||||
check("v1.5 capture stamps a default status", "status: active" in bob2, bob2)
|
||||
check("v1.5 capture seeds tags with the kind", "tags: [person]" in bob2, bob2)
|
||||
|
||||
# 11. update-capture preserves the FULL body (old code kept only line 1).
|
||||
r = subprocess.run([sys.executable, str(ECHO), "capture", "Bob Smith", "-", "--kind", "person"],
|
||||
input="met at expo\nsecond line survives\nthird line too\n",
|
||||
capture_output=True, text=True,
|
||||
env=dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21"))
|
||||
bob3 = h.ground("resources/people/bob-smith.md") or ""
|
||||
check("v1.5 update keeps the dated bullet", "- 2026-06-21: met at expo" in bob3, r.stdout + r.stderr)
|
||||
check("v1.5 update keeps EVERY body line",
|
||||
" second line survives" in bob3 and " third line too" in bob3, bob3)
|
||||
|
||||
# 12. duplicate gate: a strong fuzzy candidate STOPS creation (exit 76)...
|
||||
r = h.echo(ECHO, "capture", "Smith Consulting", "--kind", "company")
|
||||
check("v1.5 duplicate gate stops the create (exit 76)", r.returncode == 76, str(r.returncode))
|
||||
check("v1.5 gate wrote nothing", h.ground("resources/companies/smith-consulting.md") is None)
|
||||
# ... --force overrides ...
|
||||
r = h.echo(ECHO, "capture", "Smith Consulting", "--kind", "company", "--force")
|
||||
check("v1.5 --force creates past the gate",
|
||||
h.ground("resources/companies/smith-consulting.md") is not None, r.stdout + r.stderr)
|
||||
# ... and --merge-into routes the capture as an update to the canonical entity.
|
||||
r = h.echo(ECHO, "capture", "Bobby The Builder", "--kind", "person", "--merge-into", "bob-smith")
|
||||
idx4 = h.ground("_agent/index/entities.json") or ""
|
||||
check("v1.5 --merge-into updates the existing entity (mention learned as alias)",
|
||||
r.returncode == 0 and "Bobby The Builder" in idx4, r.stdout + r.stderr)
|
||||
check("v1.5 --merge-into did not spawn a new note",
|
||||
h.ground("resources/people/bobby-the-builder.md") is None)
|
||||
|
||||
# 13. recall corpus now includes session logs (down-weighted, but findable).
|
||||
h.seed("_agent/sessions/2026-06-15-1200-mpm-review.md",
|
||||
"---\ntype: session-log\ncreated: 2026-06-15\nupdated: 2026-06-15\n---\n"
|
||||
"# MPM review\n\nDiscussed the kerfuffle about invoicing.\n")
|
||||
h.echo(SWEEP, "--apply")
|
||||
r = h.echo(ECHO, "recall", "kerfuffle")
|
||||
check("v1.5 recall finds a session log by body term",
|
||||
"_agent/sessions/2026-06-15-1200-mpm-review.md" in r.stdout, r.stdout)
|
||||
rix2 = h.ground("_agent/index/recall-index.json") or ""
|
||||
check("v1.5 recall index carries doc meta (schema 2)",
|
||||
'"schema": 2' in rix2 or '"schema":2' in rix2, rix2[:200])
|
||||
|
||||
# 14. recency-aware ranking: same term, fresher note wins.
|
||||
h.seed("resources/concepts/old-idea.md",
|
||||
"---\ntype: concept\nstatus: active\ncreated: 2025-01-01\nupdated: 2025-01-01\n"
|
||||
"tags: [concept]\n---\n# Old Idea\n\nzeppelin research notes\n")
|
||||
h.seed("resources/concepts/new-idea.md",
|
||||
"---\ntype: concept\nstatus: active\ncreated: 2026-06-20\nupdated: 2026-06-20\n"
|
||||
"tags: [concept]\n---\n# New Idea\n\nzeppelin research notes\n")
|
||||
h.echo(SWEEP, "--apply")
|
||||
r = h.echo(ECHO, "recall", "zeppelin", "--json")
|
||||
try:
|
||||
env2 = json.loads(r.stdout)
|
||||
except Exception:
|
||||
env2 = {}
|
||||
primary = env2.get("primary") or []
|
||||
check("v1.5 recall --json emits structured hits",
|
||||
env2.get("ok") is True and len(primary) >= 2, r.stdout[:300])
|
||||
check("v1.5 fresher note outranks stale twin",
|
||||
primary and primary[0].get("path") == "resources/concepts/new-idea.md",
|
||||
json.dumps(primary[:2]))
|
||||
check("v1.5 recall hits carry updated/status metadata",
|
||||
primary and primary[0].get("updated") == "2026-06-20"
|
||||
and primary[0].get("status") == "active", json.dumps(primary[:1]))
|
||||
|
||||
# 15. one-tap triage: structured list, then route with an audit trail.
|
||||
h.seed("inbox/captures/inbox.md",
|
||||
"- 2026-06-01: prefers uv over pip\n- 2026-06-20: try the new espresso place\n")
|
||||
r = h.echo(ECHO, "triage", "--list", "--json")
|
||||
try:
|
||||
tenv = json.loads(r.stdout)
|
||||
except Exception:
|
||||
tenv = {}
|
||||
items = tenv.get("items") or []
|
||||
check("v1.5 triage --list parses dated captures with ages",
|
||||
len(items) == 2 and items[0].get("age_days") == 20, r.stdout[:300])
|
||||
props = [{"title": "Prefers uv over pip", "kind": "semantic",
|
||||
"body": "The operator prefers uv over pip for Python tooling.",
|
||||
"line": "- 2026-06-01: prefers uv over pip", "confidence": 0.9}]
|
||||
pfile = HERE / "_triage_props.json"
|
||||
pfile.write_text(json.dumps(props), encoding="utf-8")
|
||||
r = h.echo(ECHO, "triage", str(pfile), "--apply")
|
||||
pfile.unlink()
|
||||
routed = h.ground("_agent/memory/semantic/prefers-uv-over-pip.md")
|
||||
plog = h.ground("inbox/processing-log/2026-06-21.md") or ""
|
||||
check("v1.5 triage routes the item via capture", routed is not None, r.stdout + r.stderr)
|
||||
check("v1.5 triage writes the processing-log audit line",
|
||||
"prefers uv over pip" in plog and "_agent/memory/semantic/prefers-uv-over-pip.md" in plog, plog)
|
||||
check("v1.5 triage keeps the original capture",
|
||||
"prefers uv over pip" in (h.ground("inbox/captures/inbox.md") or ""))
|
||||
|
||||
# 16. lint flags incomplete entity frontmatter; sweep backfills it.
|
||||
h.seed("resources/companies/driftco.md",
|
||||
"---\ntype: company\ncreated: 2026-06-01\nupdated: 2026-06-01\ntags: []\n---\n# DriftCo\n")
|
||||
LINT = SCRIPTS / "vault_lint.py"
|
||||
r = h.echo(LINT)
|
||||
check("v1.5 lint flags missing status", "driftco.md: missing status" in r.stdout, r.stdout[-800:])
|
||||
check("v1.5 lint flags empty tags", "driftco.md: empty tags" in r.stdout, r.stdout[-800:])
|
||||
h.echo(SWEEP, "--apply")
|
||||
drift = h.ground("resources/companies/driftco.md") or ""
|
||||
# (the naive mock records frontmatter PATCHes as <fm:...> markers instead of
|
||||
# editing the YAML; the hi-fi suite proves the real fm semantics)
|
||||
check("v1.5 sweep backfills status",
|
||||
"status: active" in drift or '<fm:status="active">' in drift, drift)
|
||||
check("v1.5 sweep backfills the kind tag",
|
||||
"tags: [company]" in drift or '<fm:tags=["company"]>' in drift, drift)
|
||||
|
||||
# 17. hooks: stop-hook nudges once on a substantive session, then stays quiet.
|
||||
import tempfile
|
||||
tdir = Path(tempfile.mkdtemp())
|
||||
transcript = tdir / "t.jsonl"
|
||||
turns = [json.dumps({"type": "user", "message": {"content": f"user turn {i}"}}) for i in range(6)]
|
||||
transcript.write_text("\n".join(turns), encoding="utf-8")
|
||||
hook_env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_STATE_DIR=str(tdir))
|
||||
payload = json.dumps({"session_id": "s1", "transcript_path": str(transcript),
|
||||
"stop_hook_active": False})
|
||||
HOOK_STOP = SCRIPTS / "echo_hook_stop.py"
|
||||
r = subprocess.run([sys.executable, str(HOOK_STOP)], input=payload,
|
||||
capture_output=True, text=True, env=hook_env)
|
||||
try:
|
||||
henv = json.loads(r.stdout)
|
||||
except Exception:
|
||||
henv = {}
|
||||
check("v1.5 stop hook nudges a substantive session",
|
||||
r.returncode == 0 and henv.get("decision") == "block", r.stdout + r.stderr)
|
||||
r = subprocess.run([sys.executable, str(HOOK_STOP)], input=payload,
|
||||
capture_output=True, text=True, env=hook_env)
|
||||
check("v1.5 stop hook fires only once per session",
|
||||
r.returncode == 0 and not r.stdout.strip(), r.stdout)
|
||||
HOOK_START = SCRIPTS / "echo_hook_session_start.py"
|
||||
r = subprocess.run([sys.executable, str(HOOK_START)],
|
||||
input=json.dumps({"source": "startup"}),
|
||||
capture_output=True, text=True, env=hook_env)
|
||||
try:
|
||||
senv = json.loads(r.stdout)
|
||||
except Exception:
|
||||
senv = {}
|
||||
ctx = (senv.get("hookSpecificOutput") or {}).get("additionalContext", "")
|
||||
check("v1.5 session-start hook injects the load output",
|
||||
r.returncode == 0 and "marker" in ctx and "echo-vault.md" in ctx, r.stdout[:300])
|
||||
r = subprocess.run([sys.executable, str(HOOK_START)],
|
||||
input=json.dumps({"source": "resume"}),
|
||||
capture_output=True, text=True, env=hook_env)
|
||||
check("v1.5 session-start hook stays quiet on resume",
|
||||
r.returncode == 0 and not r.stdout.strip(), r.stdout)
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall feature tests passed")
|
||||
return 1 if failures else 0
|
||||
finally:
|
||||
|
||||
@@ -107,9 +107,22 @@ def main():
|
||||
g = ground(doc) or ""
|
||||
check("frontmatter replace updates existing field", "updated: 2026-06-22" in g, g)
|
||||
|
||||
# frontmatter replace on a MISSING field -> 400 (matches real API).
|
||||
# fm on a MISSING field: the server still 400s the raw PATCH (matches real API),
|
||||
# but cmd_fm (v1.5) falls back to a surgical create — the key is inserted and
|
||||
# every other line is preserved. (The old contract failed loud on exactly the
|
||||
# notes that needed repair.)
|
||||
before = ground(doc) or ""
|
||||
r = echo("fm", doc, "nonexistent_field", "x")
|
||||
check("frontmatter replace of a missing field fails loud", r.returncode != 0, r.stderr)
|
||||
check("fm creates a missing field (create-or-replace)", r.returncode == 0,
|
||||
r.stderr or r.stdout)
|
||||
g = ground(doc) or ""
|
||||
check("fm-created key holds the value", "nonexistent_field: x" in g, g)
|
||||
check("fm create preserves every existing line",
|
||||
all(line in g for line in before.splitlines() if line.strip()), g)
|
||||
|
||||
# raw `patch replace frontmatter` (no fallback) still fails loud on a missing key.
|
||||
r = echo("patch", doc, "replace", "frontmatter", "still_missing", tmp('"x"'))
|
||||
check("raw frontmatter PATCH of a missing field fails loud", r.returncode != 0, r.stderr)
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall PATCH-semantics tests passed")
|
||||
return 1 if failures else 0
|
||||
|
||||
Reference in New Issue
Block a user