ver 1.0
This commit is contained in:
jason
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""mock_olrapi_hifi.py — H4: higher-fidelity mock of the Obsidian Local REST API.
|
||||
|
||||
The shipped mock's PATCH is a "naive append": it appends the body to EOF instead of
|
||||
inserting/replacing under the target heading, so the 0.9 surface (capture's
|
||||
_append_to_existing, scope set's replace, the daily Agent-Log insert) is only ever
|
||||
validated against behavior the real API does NOT exhibit. This subclass models the
|
||||
real PATCH semantics so the eval can catch heading-insert / replace regressions.
|
||||
|
||||
Over the base mock (eval/mock_olrapi.py) this overrides do_PATCH only:
|
||||
* heading append -> insert at the END of the target section's body (before the next
|
||||
sibling/parent heading), NOT at EOF.
|
||||
* heading prepend -> insert at the START of the section body (right after the heading).
|
||||
* heading replace -> replace the section body, keep the heading line.
|
||||
* heading missing -> 400 errorCode 40080 (the silent-write-loss trigger). [unchanged]
|
||||
* frontmatter replace -> parse the YAML block and set/replace the scalar; a replace on a
|
||||
MISSING field returns 400 40080 (matches the real API, per SKILL.md).
|
||||
|
||||
Everything else (GET/PUT/POST/DELETE, directory listing, document-map, flaky/phantom
|
||||
fault injection, debug reset) is inherited unchanged from the base handler.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from http.server import ThreadingHTTPServer
|
||||
|
||||
import mock_olrapi
|
||||
from mock_olrapi import H as BaseH, STATE
|
||||
|
||||
|
||||
def _heading(line: str):
|
||||
m = re.match(r"^(#{1,6})\s+(.*)$", line)
|
||||
return (len(m.group(1)), m.group(2).strip()) if m else (0, None)
|
||||
|
||||
|
||||
def section_span(lines, target):
|
||||
"""(start, end, level) for the section under `target` (an "A::B" path; matched on its
|
||||
leaf). Body lines are lines[start+1:end]. None if the heading is absent."""
|
||||
leaf = target.split("::")[-1].strip()
|
||||
start = level = None
|
||||
for i, line in enumerate(lines):
|
||||
lv, txt = _heading(line)
|
||||
if lv and txt == leaf:
|
||||
start, level = i, lv
|
||||
break
|
||||
if start is None:
|
||||
return None
|
||||
end = len(lines)
|
||||
for j in range(start + 1, len(lines)):
|
||||
lv, _ = _heading(lines[j])
|
||||
if lv and lv <= level:
|
||||
end = j
|
||||
break
|
||||
return start, end, level
|
||||
|
||||
|
||||
def set_frontmatter(text: str, field: str, raw_value: str):
|
||||
"""Return (new_text, existed). raw_value is the JSON the client PATCHed."""
|
||||
try:
|
||||
value = json.loads(raw_value)
|
||||
except Exception:
|
||||
value = raw_value.strip().strip('"')
|
||||
rendered = value if isinstance(value, str) else json.dumps(value)
|
||||
new_line = f"{field}: {rendered}"
|
||||
if not text.startswith("---"):
|
||||
return f"---\n{new_line}\n---\n\n{text}", False
|
||||
end = text.find("\n---", 3)
|
||||
head = text[4:end] if end != -1 else text
|
||||
rest = text[end:] if end != -1 else "\n---\n"
|
||||
out, existed = [], False
|
||||
for line in head.splitlines():
|
||||
if re.match(rf"^{re.escape(field)}\s*:", line):
|
||||
out.append(new_line)
|
||||
existed = True
|
||||
else:
|
||||
out.append(line)
|
||||
if not existed:
|
||||
out.append(new_line)
|
||||
return "---\n" + "\n".join(out) + rest, existed
|
||||
|
||||
|
||||
class H(BaseH):
|
||||
def do_PATCH(self):
|
||||
vp = self._vpath()
|
||||
body = self._read_body()
|
||||
if vp is None:
|
||||
return self._json(400, {"message": "bad path"})
|
||||
ttype = self.headers.get("Target-Type", "")
|
||||
op = self.headers.get("Operation", "append")
|
||||
target = self.headers.get("Target", "")
|
||||
cur = STATE.get(vp, "")
|
||||
|
||||
if ttype == "heading":
|
||||
lines = cur.split("\n")
|
||||
span = section_span(lines, target)
|
||||
if span is None:
|
||||
return self._json(400, {"errorCode": 40080, "message": "invalid-target"})
|
||||
start, end, _ = span
|
||||
payload = body.strip("\n").split("\n") if body.strip("\n") else [""]
|
||||
if op == "replace":
|
||||
lines[start + 1:end] = payload
|
||||
elif op == "prepend":
|
||||
lines[start + 1:start + 1] = payload
|
||||
else: # append at the tail of the section body (before the next heading)
|
||||
ins = end
|
||||
while ins > start + 1 and lines[ins - 1].strip() == "":
|
||||
ins -= 1
|
||||
lines[ins:ins] = payload
|
||||
STATE[vp] = "\n".join(lines)
|
||||
return self._send(200, "")
|
||||
|
||||
if ttype == "frontmatter":
|
||||
new, existed = set_frontmatter(cur, target, body)
|
||||
if op == "replace" and not existed:
|
||||
return self._json(400, {"errorCode": 40080, "message": "invalid-target"})
|
||||
STATE[vp] = new
|
||||
return self._send(200, "")
|
||||
|
||||
STATE[vp] = cur + "\n" + body
|
||||
return self._send(200, "")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=8800)
|
||||
a = ap.parse_args()
|
||||
srv = ThreadingHTTPServer(("127.0.0.1", a.port), H)
|
||||
print(f"mock-olrapi-hifi listening on http://127.0.0.1:{a.port}", flush=True)
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -33,7 +33,7 @@ from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
|
||||
KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
|
||||
# ----- tiny HTTP helpers (used for setup, ground truth, and the 0.6 model) -----
|
||||
def http(method, url, body=None, headers=None):
|
||||
|
||||
+47
-2
@@ -14,7 +14,7 @@ HERE = Path(__file__).resolve().parent
|
||||
SCRIPTS = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts"
|
||||
ECHO = SCRIPTS / "echo.py"
|
||||
SWEEP = SCRIPTS / "sweep.py"
|
||||
KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
|
||||
failures = []
|
||||
|
||||
@@ -102,6 +102,12 @@ def main():
|
||||
r = h.echo(ECHO, "recall", "MPM")
|
||||
check("recall surfaces linked person", "resources/people/bob-smith.md" in r.stdout, r.stdout)
|
||||
|
||||
# 4b. BM25 finds a note by a BODY term. /search/simple is mocked empty, so a hit
|
||||
# here can only come from the local BM25 recall index (proves H1 lexical layer).
|
||||
r = h.echo(ECHO, "recall", "principal")
|
||||
check("recall finds note by body term (BM25)",
|
||||
"resources/companies/mpm.md" in r.stdout, r.stdout)
|
||||
|
||||
# 5. explicit link between two fresh notes
|
||||
h.echo(ECHO, "capture", "Alpha", "--kind", "concept")
|
||||
h.echo(ECHO, "capture", "Beta", "--kind", "concept")
|
||||
@@ -120,10 +126,49 @@ def main():
|
||||
r = h.echo(SWEEP, "--apply")
|
||||
marker = h.ground("_agent/echo-vault.md")
|
||||
check("sweep runs clean", r.returncode == 0, r.stdout + r.stderr)
|
||||
check("sweep stamps schema 3", marker and "schema_version=3" in marker.replace(": ", "="), marker)
|
||||
check("sweep stamps schema 4", marker and "schema_version=4" in marker.replace(": ", "="), marker)
|
||||
idx2 = h.ground("_agent/index/entities.json")
|
||||
check("sweep rebuilt index has all entities",
|
||||
idx2 and all(s in idx2 for s in ["bob-smith", "mpm", "alpha", "beta"]))
|
||||
rix = h.ground("_agent/index/recall-index.json")
|
||||
check("sweep builds the recall (BM25) index",
|
||||
rix is not None and '"postings"' in rix and "principal" in rix, rix)
|
||||
|
||||
# 8. H3 — atomic_index_update re-reads fresh and MERGES, so an entry written by a
|
||||
# "concurrent" session survives our write; and the advisory lock is released after.
|
||||
h.http("PUT", f"{base}/vault/_agent/index/entities.json",
|
||||
'{"schema":1,"updated":"2026-06-21","entities":{"ext-entity":'
|
||||
'{"path":"resources/concepts/ext-entity.md","kind":"concept","title":"Ext",'
|
||||
'"aliases":[],"last_seen":"2026-06-21"}}}')
|
||||
env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_TODAY="2026-06-21", PYTHONPATH=str(SCRIPTS))
|
||||
snippet = ("import echo_index as ix, echo_concurrency as c; "
|
||||
"c.atomic_index_update(lambda d: ix.upsert(d,'mine',"
|
||||
"'resources/concepts/mine.md','concept','Mine'))")
|
||||
subprocess.run([sys.executable, "-c", snippet], env=env, capture_output=True, text=True)
|
||||
idx3 = h.ground("_agent/index/entities.json")
|
||||
check("H3 atomic update merges a concurrent entry (no clobber)",
|
||||
bool(idx3) and "ext-entity" in idx3 and "mine" in idx3, idx3)
|
||||
check("H3 advisory lock released after the transaction",
|
||||
h.ground("_agent/locks/vault.lock") is None)
|
||||
|
||||
# 9. M4 — --dry-run previews and writes nothing; --json emits a parseable envelope.
|
||||
r = h.echo(ECHO, "capture", "DryRun Co", "--kind", "company", "--dry-run", "--json")
|
||||
try:
|
||||
env = json.loads(r.stdout)
|
||||
except Exception:
|
||||
env = {}
|
||||
check("M4 --dry-run emits a dry-run JSON envelope",
|
||||
env.get("action", "").startswith("dry-run") and env.get("path") == "resources/companies/dryrun-co.md", r.stdout)
|
||||
check("M4 --dry-run writes nothing", h.ground("resources/companies/dryrun-co.md") is None)
|
||||
r = h.echo(ECHO, "capture", "Json Co", "--kind", "company", "--json")
|
||||
try:
|
||||
env = json.loads(r.stdout)
|
||||
except Exception:
|
||||
env = {}
|
||||
check("M4 --json envelope reports created + path",
|
||||
env.get("ok") is True and env.get("action") == "created"
|
||||
and env.get("path") == "resources/companies/json-co.md", r.stdout)
|
||||
check("M4 --json capture actually wrote the note", h.ground("resources/companies/json-co.md") is not None)
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall feature tests passed")
|
||||
return 1 if failures else 0
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""test_offline_queue.py — H2: writes survive an outage; cold start degrades to cache.
|
||||
|
||||
Phase 1 — vault DOWN: `put` / `append` are queued (exit 0, "queued"), not lost.
|
||||
Phase 2 — vault UP: `flush` replays them idempotently; the writes land.
|
||||
Phase 3 — vault UP: `load` caches the orientation reads.
|
||||
Phase 4 — vault DOWN: `load` serves the last-known-good cache and says OFFLINE.
|
||||
|
||||
Drives the real echo.py against eval/mock_olrapi.py. No creds, no live vault.
|
||||
Run: python test_offline_queue.py [--port 8840]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
DEAD = "http://127.0.0.1:59998" # nothing listens here -> connection refused
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def http(method, url, body=None):
|
||||
data = body.encode() if isinstance(body, str) else body
|
||||
req = urllib.request.Request(url, data=data, method=method,
|
||||
headers={"Authorization": f"Bearer {KEY}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return r.status, r.read().decode("utf-8", "replace")
|
||||
except Exception as e: # noqa: BLE001
|
||||
return getattr(e, "code", 0), ""
|
||||
|
||||
|
||||
def tmp(content):
|
||||
f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
|
||||
f.write(content); f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=8840)
|
||||
a = ap.parse_args()
|
||||
mock_base = f"http://127.0.0.1:{a.port}"
|
||||
state = tempfile.mkdtemp()
|
||||
|
||||
def echo(base, *args):
|
||||
env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_STATE_DIR=state,
|
||||
ECHO_VERIFY="0", ECHO_TODAY="2026-06-22")
|
||||
return subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env)
|
||||
|
||||
def start_mock():
|
||||
p = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
for _ in range(50):
|
||||
try:
|
||||
urllib.request.urlopen(f"{mock_base}/__debug__reset", data=b"", timeout=1); break
|
||||
except Exception:
|
||||
time.sleep(0.1)
|
||||
return p
|
||||
|
||||
def ground(path):
|
||||
_, body = http("GET", f"{mock_base}/__debug__?path={path}")
|
||||
return None if body == "<<MISSING>>" else body
|
||||
|
||||
srv = None
|
||||
try:
|
||||
# --- Phase 1: vault DOWN -> writes are queued, not lost ------------------
|
||||
r = echo(DEAD, "put", "_agent/sessions/2026-06-22-1200-x.md", tmp("EVALMARK-session\n"))
|
||||
check("offline put exits 0 (queued)", r.returncode == 0, r.stderr)
|
||||
check("offline put reports queued", "queued (offline)" in r.stdout, r.stdout)
|
||||
r = echo(DEAD, "append", "inbox/captures/inbox.md", "- 2026-06-22: EVALMARK-inbox")
|
||||
check("offline append reports queued", "queued (offline)" in r.stdout, r.stdout)
|
||||
outbox = Path(state) / "outbox.ndjson"
|
||||
check("two writes are persisted to the outbox",
|
||||
outbox.exists() and len(outbox.read_text(encoding="utf-8").splitlines()) == 2)
|
||||
|
||||
# --- Phase 2: vault UP -> flush replays them -----------------------------
|
||||
srv = start_mock()
|
||||
r = echo(mock_base, "flush")
|
||||
check("flush replays the queue", "flushed 2" in r.stdout, r.stdout + r.stderr)
|
||||
check("queued PUT landed in the vault", (ground("_agent/sessions/2026-06-22-1200-x.md") or "").find("EVALMARK-session") >= 0)
|
||||
check("queued APPEND landed in the vault", "EVALMARK-inbox" in (ground("inbox/captures/inbox.md") or ""))
|
||||
check("outbox is empty after a clean flush", not outbox.exists() or not outbox.read_text(encoding="utf-8").strip())
|
||||
|
||||
# idempotent replay: a second flush of the same content must not duplicate.
|
||||
echo(DEAD, "append", "inbox/captures/inbox.md", "- 2026-06-22: EVALMARK-inbox") # re-queue same line
|
||||
echo(mock_base, "flush")
|
||||
inbox = ground("inbox/captures/inbox.md") or ""
|
||||
check("replay is idempotent (no duplicate line)", inbox.count("EVALMARK-inbox") == 1, inbox)
|
||||
|
||||
# --- Phase 3: vault UP -> load caches the orientation reads ---------------
|
||||
http("PUT", f"{mock_base}/vault/_agent/echo-vault.md", "schema_version: 4\nEVALMARK-marker\n")
|
||||
r = echo(mock_base, "load")
|
||||
check("online load shows marker", "EVALMARK-marker" in r.stdout, r.stdout)
|
||||
check("load wrote a cache dir", (Path(state) / "cache").exists())
|
||||
|
||||
# --- Phase 4: vault DOWN -> load serves the cache ------------------------
|
||||
srv.terminate(); srv.wait(timeout=5); srv = None
|
||||
r = echo(DEAD, "load")
|
||||
check("offline load flags OFFLINE", "OFFLINE" in r.stdout, r.stdout)
|
||||
check("offline load serves cached marker", "EVALMARK-marker" in r.stdout, r.stdout)
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall offline-queue tests passed")
|
||||
return 1 if failures else 0
|
||||
finally:
|
||||
if srv:
|
||||
srv.terminate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""test_patch_semantics.py — H4: prove the hi-fi mock models REAL PATCH semantics.
|
||||
|
||||
These are exactly the behaviors the shipped naive mock cannot reproduce (it appends
|
||||
everything at EOF), so without this the section-replace / section-append / missing-
|
||||
heading / frontmatter-replace paths are untested. Drives the real echo.py against
|
||||
mock_olrapi_hifi.py.
|
||||
|
||||
Run: python test_patch_semantics.py [--port 8820]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def http(method, url, body=None, headers=None):
|
||||
data = body.encode() if isinstance(body, str) else body
|
||||
req = urllib.request.Request(url, data=data, method=method,
|
||||
headers={"Authorization": f"Bearer {KEY}", **(headers or {})})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return r.status, r.read().decode("utf-8", "replace")
|
||||
except Exception as e: # noqa: BLE001
|
||||
return getattr(e, "code", 0), ""
|
||||
|
||||
|
||||
def tmp(content):
|
||||
f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False, encoding="utf-8")
|
||||
f.write(content)
|
||||
f.close()
|
||||
return f.name
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=8820)
|
||||
a = ap.parse_args()
|
||||
base = f"http://127.0.0.1:{a.port}"
|
||||
srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi_hifi.py"), "--port", str(a.port)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
|
||||
def echo(*args):
|
||||
env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_VERIFY="0")
|
||||
return subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env)
|
||||
|
||||
def ground(path):
|
||||
_, body = http("GET", f"{base}/__debug__?path={path}")
|
||||
return None if body == "<<MISSING>>" else body
|
||||
|
||||
try:
|
||||
for _ in range(50):
|
||||
try:
|
||||
urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1)
|
||||
break
|
||||
except Exception:
|
||||
time.sleep(0.1)
|
||||
|
||||
doc = "doc.md"
|
||||
seed = ("---\ntype: note\nupdated: 2026-01-01\n---\n\n"
|
||||
"# Doc\n\n## Scope\nold scope\n\n## Other\nkeep me\n")
|
||||
|
||||
# replace: Scope body becomes new; Other untouched.
|
||||
http("PUT", f"{base}/vault/{doc}", seed)
|
||||
echo("patch", doc, "replace", "heading", "Doc::Scope", tmp("new scope"))
|
||||
g = ground(doc) or ""
|
||||
check("heading replace swaps section body", "new scope" in g and "old scope" not in g, g)
|
||||
check("heading replace leaves sibling section intact", "## Other\nkeep me" in g, g)
|
||||
|
||||
# append: lands INSIDE the section (before '## Other'), not at EOF.
|
||||
echo("patch", doc, "append", "heading", "Doc::Scope", tmp("appended line"))
|
||||
g = ground(doc) or ""
|
||||
check("heading append inserts within the section (not EOF)",
|
||||
"appended line" in g and g.index("appended line") < g.index("## Other"), g)
|
||||
|
||||
# prepend: lands at the TOP of the section body.
|
||||
echo("patch", doc, "prepend", "heading", "Doc::Scope", tmp("first line"))
|
||||
g = ground(doc) or ""
|
||||
check("heading prepend inserts at section top",
|
||||
"first line" in g and g.index("first line") < g.index("new scope"), g)
|
||||
|
||||
# missing heading -> 400 -> echo.py exits non-zero (the silent-loss guard).
|
||||
r = echo("patch", doc, "append", "heading", "Doc::Nope", tmp("lost?"))
|
||||
check("missing heading fails loud (non-zero exit)", r.returncode != 0, r.stderr)
|
||||
check("missing heading does not write", "lost?" not in (ground(doc) or ""))
|
||||
|
||||
# frontmatter replace on an existing field updates it.
|
||||
echo("fm", doc, "updated", "2026-06-22")
|
||||
g = ground(doc) or ""
|
||||
check("frontmatter replace updates existing field", "updated: 2026-06-22" in g, g)
|
||||
|
||||
# frontmatter replace on a MISSING field -> 400 (matches real API).
|
||||
r = echo("fm", doc, "nonexistent_field", "x")
|
||||
check("frontmatter replace of a missing field fails loud", r.returncode != 0, r.stderr)
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall PATCH-semantics tests passed")
|
||||
return 1 if failures else 0
|
||||
finally:
|
||||
srv.terminate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""test_reflect.py — H5: session-reflection capture (dry-run vs --apply).
|
||||
|
||||
A dry-run previews and writes NOTHING; --apply routes each proposal through capture
|
||||
(creating notes, the inbox line, and skipping low-confidence items). Drives the real
|
||||
echo.py against eval/mock_olrapi.py. No creds, no live vault.
|
||||
|
||||
Run: python test_reflect.py [--port 8850]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
|
||||
KEY = "test-key-not-a-real-secret"
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(name, cond, detail=""):
|
||||
print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=8850)
|
||||
a = ap.parse_args()
|
||||
base = f"http://127.0.0.1:{a.port}"
|
||||
srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
|
||||
def http(method, url, body=None):
|
||||
data = body.encode() if isinstance(body, str) else body
|
||||
req = urllib.request.Request(url, data=data, method=method,
|
||||
headers={"Authorization": f"Bearer {KEY}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return r.status, r.read().decode("utf-8", "replace")
|
||||
except Exception as e: # noqa: BLE001
|
||||
return getattr(e, "code", 0), ""
|
||||
|
||||
def echo(*args, stdin=None):
|
||||
env = dict(os.environ, ECHO_BASE=base, ECHO_KEY=KEY, ECHO_VERIFY="1", ECHO_TODAY="2026-06-22")
|
||||
return subprocess.run([sys.executable, str(ECHO), *args], input=stdin,
|
||||
capture_output=True, text=True, env=env)
|
||||
|
||||
def ground(path):
|
||||
_, body = http("GET", f"{base}/__debug__?path={path}")
|
||||
return None if body == "<<MISSING>>" else body
|
||||
|
||||
try:
|
||||
for _ in range(50):
|
||||
try:
|
||||
urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break
|
||||
except Exception:
|
||||
time.sleep(0.1)
|
||||
http("PUT", f"{base}/vault/_agent/echo-vault.md", "---\nschema_version: 4\n---\n# marker\n")
|
||||
|
||||
proposals = [
|
||||
{"title": "Acme Corp", "kind": "company", "body": "A vendor ECHO integrates with.", "confidence": 0.9},
|
||||
{"title": "Use uv not pip", "kind": "semantic", "body": "Jason standardizes on uv.", "confidence": 0.95},
|
||||
{"title": "half-formed idea", "inbox": True, "confidence": 0.9},
|
||||
{"title": "Maybe relevant", "kind": "concept", "confidence": 0.2}, # below floor -> skipped
|
||||
]
|
||||
pfile = tempfile.NamedTemporaryFile("w", suffix=".json", delete=False, encoding="utf-8")
|
||||
json.dump(proposals, pfile); pfile.close()
|
||||
|
||||
# dry-run: previews, writes nothing
|
||||
r = echo("reflect", pfile.name)
|
||||
check("dry-run previews", "dry-run" in r.stdout and "Acme Corp" in r.stdout, r.stdout)
|
||||
check("dry-run drops the low-confidence proposal", "Maybe relevant" not in r.stdout, r.stdout)
|
||||
check("dry-run writes nothing", ground("resources/companies/acme-corp.md") is None)
|
||||
|
||||
# --apply: routes each through capture
|
||||
r = echo("reflect", pfile.name, "--apply")
|
||||
check("apply reports applied count", "applied" in r.stdout, r.stdout + r.stderr)
|
||||
check("apply creates the company note",
|
||||
(ground("resources/companies/acme-corp.md") or "").find("type: company") >= 0)
|
||||
check("apply creates the semantic note",
|
||||
ground("_agent/memory/semantic/use-uv-not-pip.md") is not None)
|
||||
check("apply routes the inbox proposal", "half-formed idea" in (ground("inbox/captures/inbox.md") or ""))
|
||||
check("apply still skips the low-confidence proposal",
|
||||
ground("resources/concepts/maybe-relevant.md") is None)
|
||||
|
||||
# stdin path also works (proposals piped, not a file)
|
||||
r = echo("reflect", "-", stdin=json.dumps([{"title": "Piped Co", "kind": "company", "confidence": 0.9}]))
|
||||
check("reflect reads proposals from stdin", "Piped Co" in r.stdout, r.stdout + r.stderr)
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall reflect tests passed")
|
||||
return 1 if failures else 0
|
||||
finally:
|
||||
srv.terminate()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user