#!/usr/bin/env python3
"""run_eval.py — A/B eval of echo-memory 0.6 (raw curl, documented recipes) vs
0.7 (the shipped echo.py client) over representative memory operations, with
fault injection.

Design
------
* A mock Obsidian REST API (mock_olrapi.py) gives deterministic behavior + faults,
  so no credentials are needed and the real vault is never touched.
* The 0.7 side runs the ACTUAL shipped scripts/echo.py (status-checked, retry, verify,
  idempotent append).
* The 0.6 side faithfully models the documented raw-curl recipes: it performs the
  same HTTP but does NOT inspect status, does NOT retry, does NOT verify, and does
  NOT dedupe — exactly what SKILL.md 0.6 told the model to emit.
* Ground truth is read back independently from the mock after each method, so a
  "silent failure" (method reported success but the vault is actually wrong, and
  nobody noticed) is detected regardless of what the method claimed.
* Each (scenario, method) runs against a freshly reset + re-seeded server, so faults
  (e.g. one-time 503) are identical for both methods.

Metrics
-------
* gen_tokens     : output tokens the MODEL must generate for the op (len(emitted)/CPT).
* silent_failure : claimed success BUT ground truth is wrong (lost write or duplicate).
* detected       : the method surfaced the failure (loud, retryable) instead of hiding it.
* effective_tokens = gen_tokens + silent_failures*RECOVERY + detected*DETECT_COST
                     (RECOVERY/DETECT_COST are labeled assumptions, tune via env).

Usage:  python3 run_eval.py [--port 8799] [--cpt 4] [--recovery 1500] [--detect-cost 80]
"""
import os, sys, json, time, subprocess, tempfile, argparse, urllib.request, urllib.error
from pathlib import Path

HERE = Path(__file__).resolve().parent
ECHO = HERE.parent / "echo-memory.plugin.src" / "skills" / "echo-memory" / "scripts" / "echo.py"
KEY  = "test-key-not-a-real-secret"

# ----- tiny HTTP helpers (used for setup, ground truth, and the 0.6 model) -----
def http(method, url, body=None, headers=None):
    data = body.encode() if isinstance(body, str) else body
    req = urllib.request.Request(url, data=data, method=method,
                                 headers={"Authorization": f"Bearer {KEY}", **(headers or {})})
    try:
        with urllib.request.urlopen(req, timeout=10) as r:
            return r.status, r.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        return e.code, e.read().decode("utf-8", "replace")
    except Exception as e:
        return 0, str(e)

class Eval:
    def __init__(self, base, cpt):
        self.base, self.cpt = base, cpt
    def reset(self): http("POST", f"{self.base}/__debug__reset")
    def seed(self, path, content): http("PUT", f"{self.base}/vault/{path}", content)
    def ground(self, path):
        st, body = http("GET", f"{self.base}/__debug__?path={path}")
        return None if body == "<<MISSING>>" else body
    def toks(self, text): return round(len(text) / self.cpt)
    def echo(self, *args):
        env = dict(os.environ, ECHO_BASE=self.base, ECHO_KEY=KEY, ECHO_VERIFY="1", ECHO_LOCK_TTL="900")
        p = subprocess.run([sys.executable, str(ECHO), *args], capture_output=True, text=True, env=env)
        return p.returncode

# ---- faithful 0.6 recipe text (what the model emitted), for token accounting ----
def recipe_get(path):
    return (f'curl -s -H "Authorization: Bearer {KEY}" '
            f'"https://echoapi.alwisp.com/vault/{path}"')
def recipe_put(path):
    return (f"cat > /tmp/obs_file.md << 'EOF'\n<body>\nEOF\n\n"
            f'curl -s -X PUT -H "Authorization: Bearer {KEY}" '
            f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_file.md '
            f'"https://echoapi.alwisp.com/vault/{path}"')
def recipe_post(path):
    return (f"cat > /tmp/obs_entry.md << 'EOF'\n<line>\nEOF\n\n"
            f'curl -s -X POST -H "Authorization: Bearer {KEY}" '
            f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_entry.md '
            f'"https://echoapi.alwisp.com/vault/{path}"')
def recipe_patch(path, target):
    return (f"cat > /tmp/obs_patch.md << 'EOF'\n<body>\nEOF\n\n"
            f'curl -s -X PATCH -H "Authorization: Bearer {KEY}" '
            f'-H "Operation: append" -H "Target-Type: heading" -H "Target: {target}" '
            f'-H "Content-Type: text/markdown" --data-binary @/tmp/obs_patch.md '
            f'"https://echoapi.alwisp.com/vault/{path}"')

def tmpfile(content):
    f = tempfile.NamedTemporaryFile("w", suffix=".md", delete=False); f.write(content); f.close()
    return f.name

# ---------------------------------------------------------------------------
# Scenarios. Each defines seed(), and a run for each method that returns:
#   {emit: <text model generates>, claimed_ok: bool, detected: bool}
# plus a ground-truth check producing {persisted, duplicates}.
# ---------------------------------------------------------------------------
def scenarios(ev):
    out = []

    # S1 — agent-log append where the heading is MISSING (-> 400 invalid-target)
    def s1():
        path, tgt = "journal/daily/2026-06-19.md", "2026-06-19::Agent Log"
        line = "- 2026-06-19: EVALMARK-s1"
        def seed(): ev.seed(path, "---\ntype: daily-note\n---\n\n# 2026-06-19\n\n## Notes\n")
        def m06():
            http("PATCH", f"{ev.base}/vault/{path}", line,
                 {"Operation": "append", "Target-Type": "heading", "Target": tgt})  # status ignored
            return {"emit": recipe_patch(path, tgt), "claimed_ok": True, "detected": False}
        def m07():
            rc = ev.echo("patch", path, "append", "heading", tgt, tmpfile(line))
            return {"emit": f'"$ECHO" patch {path} append heading "{tgt}" /tmp/x.md',
                    "claimed_ok": rc == 0, "detected": rc != 0}
        def gt():
            c = ev.ground(path) or ""
            return {"persisted": "EVALMARK-s1" in c, "duplicates": max(0, c.count("EVALMARK-s1") - 1)}
        return dict(name="agent-log-missing-heading", fault="bad heading -> 400 invalid-target",
                    seed=seed, m06=m06, m07=m07, gt=gt)
    out.append(s1())

    # S2 — scope switch (no fault; pure token comparison on a PATCH replace)
    def s2():
        path, tgt = "_agent/context/current-context.md", "Current Context::Scope"
        def seed(): ev.seed(path, "---\ntype: context-bundle\n---\n\n# Current Context\n\n## Scope\nold scope\n")
        def m06():
            http("PATCH", f"{ev.base}/vault/{path}", "EVALMARK-s2 new scope",
                 {"Operation": "replace", "Target-Type": "heading", "Target": tgt})
            return {"emit": recipe_patch(path, tgt), "claimed_ok": True, "detected": False}
        def m07():
            rc = ev.echo("patch", path, "replace", "heading", tgt, tmpfile("EVALMARK-s2 new scope"))
            return {"emit": f'"$ECHO" patch {path} replace heading "{tgt}" /tmp/x.md',
                    "claimed_ok": rc == 0, "detected": rc != 0}
        def gt():
            c = ev.ground(path) or ""
            return {"persisted": "EVALMARK-s2" in c, "duplicates": 0}
        return dict(name="scope-switch", fault="(none)", seed=seed, m06=m06, m07=m07, gt=gt)
    out.append(s2())

    # S3 — inbox capture issued TWICE (retry/replay): dedup vs duplicate line
    def s3():
        path = "inbox/captures/inbox.md"; line = "- 2026-06-19: EVALMARK-s3"
        def seed(): ev.seed(path, "# Inbox\n")
        def m06():
            http("POST", f"{ev.base}/vault/{path}", line + "\n")  # no idempotency check
            http("POST", f"{ev.base}/vault/{path}", line + "\n")
            return {"emit": recipe_post(path) + "\n# (repeated on retry)\n" + recipe_post(path),
                    "claimed_ok": True, "detected": False}
        def m07():
            rc1 = ev.echo("append", path, line)
            rc2 = ev.echo("append", path, line)  # second is skipped by read-before-POST
            return {"emit": f'"$ECHO" append {path} "{line}"\n"$ECHO" append {path} "{line}"',
                    "claimed_ok": rc1 == 0 and rc2 == 0, "detected": False}
        def gt():
            c = ev.ground(path) or ""
            return {"persisted": "EVALMARK-s3" in c, "duplicates": max(0, c.count("EVALMARK-s3") - 1)}
        return dict(name="inbox-capture-replayed", fault="duplicate attempt (retry/replay)",
                    seed=seed, m06=m06, m07=m07, gt=gt)
    out.append(s3())

    # S4 — session-log PUT under a flaky network (one-time 503 then success)
    def s4():
        path = "_agent/sessions/2026-06-19-1430-flaky-eval.md"
        body = "---\ntype: session-log\n---\n\n# Session\nEVALMARK-s4\n"
        def seed(): pass  # file does not pre-exist
        def m06():
            http("PUT", f"{ev.base}/vault/{path}", body)  # single shot, no retry, status ignored
            return {"emit": recipe_put(path), "claimed_ok": True, "detected": False}
        def m07():
            rc = ev.echo("put", path, tmpfile(body))  # echo.py retries the 503 once
            return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0}
        def gt():
            c = ev.ground(path) or ""
            return {"persisted": "EVALMARK-s4" in c, "duplicates": 0}
        return dict(name="session-log-flaky-network", fault="one-time 503 (transient)",
                    seed=seed, m06=m06, m07=m07, gt=gt)
    out.append(s4())

    # S5 — heartbeat PUT accepted-but-not-persisted (proxy hiccup): verify catches it
    def s5():
        path = "_agent/heartbeat/phantom-eval.md"; body = "EVALMARK-s5 @ 2026-06-19T14:30:00Z\n"
        def seed(): pass
        def m06():
            http("PUT", f"{ev.base}/vault/{path}", body)  # 200 returned; no read-back verify
            return {"emit": recipe_put(path), "claimed_ok": True, "detected": False}
        def m07():
            rc = ev.echo("put", path, tmpfile(body))  # verify GET -> 404 -> die
            return {"emit": f'"$ECHO" put {path} /tmp/x.md', "claimed_ok": rc == 0, "detected": rc != 0}
        def gt():
            c = ev.ground(path) or ""
            return {"persisted": "EVALMARK-s5" in c, "duplicates": 0}
        return dict(name="heartbeat-phantom-write", fault="accepted-but-not-persisted",
                    seed=seed, m06=m06, m07=m07, gt=gt)
    out.append(s5())

    # S6 — cold-start load: 6 reads (no fault; pure token comparison)
    def s6():
        paths = ["_agent/echo-vault.md", "_agent/memory/semantic/operator-preferences.md",
                 "_agent/context/current-context.md", "_agent/heartbeat/last-session.md",
                 "journal/daily/2026-06-19.md", "inbox/captures/inbox.md"]
        def seed():
            for p in paths: ev.seed(p, f"# {p}\nEVALMARK-s6\n")
        def m06():
            for p in paths: http("GET", f"{ev.base}/vault/{p}")
            return {"emit": "\n".join(recipe_get(p) for p in paths), "claimed_ok": True, "detected": False}
        def m07():
            for p in paths: ev.echo("get", p)
            return {"emit": "\n".join(f'"$ECHO" get {p}' for p in paths), "claimed_ok": True, "detected": False}
        def gt(): return {"persisted": True, "duplicates": 0}
        return dict(name="cold-start-load-6-reads", fault="(none)", seed=seed, m06=m06, m07=m07, gt=gt)
    out.append(s6())

    return out

def run_method(ev, scn, key):
    ev.reset(); scn["seed"]()
    res = scn[key]()
    g = scn["gt"]()
    # a write op "should persist"; reads (s6) and replays are judged by their own gt
    bad = (not g["persisted"] and scn["name"] != "cold-start-load-6-reads") or g["duplicates"] > 0
    silent = bool(res["claimed_ok"] and bad)
    return {"emit_tokens": ev.toks(res["emit"]), "claimed_ok": res["claimed_ok"],
            "detected": res["detected"], "persisted": g["persisted"],
            "duplicates": g["duplicates"], "silent_failure": silent}

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--port", type=int, default=8799)
    ap.add_argument("--cpt", type=float, default=4.0, help="chars per token")
    ap.add_argument("--recovery", type=int, default=1500, help="token penalty per silent failure (recovery loop)")
    ap.add_argument("--detect-cost", type=int, default=80, help="token cost to re-issue a corrected call after a detected failure")
    a = ap.parse_args()

    base = f"http://127.0.0.1:{a.port}"
    srv = subprocess.Popen([sys.executable, str(HERE / "mock_olrapi.py"), "--port", str(a.port)],
                           stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
    try:
        for _ in range(50):  # wait for ready
            try:
                urllib.request.urlopen(f"{base}/__debug__reset", data=b"", timeout=1); break
            except Exception: time.sleep(0.1)
        ev = Eval(base, a.cpt)
        rows, agg = [], {"06": {}, "07": {}}
        for scn in scenarios(ev):
            r06 = run_method(ev, scn, "m06")
            r07 = run_method(ev, scn, "m07")
            rows.append({"scenario": scn["name"], "fault": scn["fault"], "06": r06, "07": r07})

        def totals(side):
            gen = sum(r[side]["emit_tokens"] for r in rows)
            sil = sum(1 for r in rows if r[side]["silent_failure"])
            det = sum(1 for r in rows if r[side]["detected"])
            dup = sum(r[side]["duplicates"] for r in rows)
            eff = gen + sil * a.recovery + det * a.detect_cost
            return dict(gen_tokens=gen, silent_failures=sil, detected_failures=det,
                        duplicates=dup, effective_tokens=eff)
        T06, T07 = totals("06"), totals("07")

        # ---- report ----
        line = "=" * 78
        print(f"\n{line}\nECHO MEMORY — 0.6 vs 0.7 A/B EVAL")
        print(f"(mock OLRAPI; chars/token={a.cpt}, recovery-penalty={a.recovery}, detect-cost={a.detect_cost})\n{line}")
        hdr = f"{'scenario':28} {'fault':32} {'gen06':>5} {'gen07':>5} {'silent06':>8} {'silent07':>8}"
        print(hdr); print("-" * len(hdr))
        for r in rows:
            print(f"{r['scenario']:28} {r['fault']:32} "
                  f"{r['06']['emit_tokens']:>5} {r['07']['emit_tokens']:>5} "
                  f"{('YES' if r['06']['silent_failure'] else '-'):>8} "
                  f"{('YES' if r['07']['silent_failure'] else '-'):>8}")
        print("-" * len(hdr))

        def pct(old, new): return f"{(old-new)/old*100:+.0f}%" if old else "n/a"
        print(f"\n{'metric':30} {'0.6':>10} {'0.7':>10} {'delta':>10}")
        print("-" * 62)
        for label, k in [("generated tokens", "gen_tokens"),
                         ("silent failures", "silent_failures"),
                         ("duplicate lines", "duplicates"),
                         ("detected (loud) failures", "detected_failures"),
                         ("effective tokens (incl. recovery)", "effective_tokens")]:
            o, n = T06[k], T07[k]
            d = pct(o, n) if "token" in k else f"{n-o:+d}"
            print(f"{label:30} {o:>10} {n:>10} {d:>10}")
        wrows = [r for r in rows if r['scenario'] != 'cold-start-load-6-reads']
        denom = len(wrows)
        sf06 = sum(1 for r in wrows if not r['06']['silent_failure'])   # silent-error-free
        sf07 = sum(1 for r in wrows if not r['07']['silent_failure'])
        pr06 = sum(1 for r in wrows if r['06']['persisted'])            # write actually landed
        pr07 = sum(1 for r in wrows if r['07']['persisted'])
        print("-" * 62)
        print(f"{'silent-error-free ops':30} {str(sf06)+'/'+str(denom):>10} {str(sf07)+'/'+str(denom):>10}")
        print(f"{'writes actually persisted':30} {str(pr06)+'/'+str(denom):>10} {str(pr07)+'/'+str(denom):>10}")
        print(f"  (the 2 un-persisted 0.7 ops are bad-heading + phantom: not persistable in a single")
        print(f"   call, but 0.7 fails LOUD so the agent's ensure-heading/retry path can recover.)")
        print(f"\nNOTE: recovery-penalty and detect-cost are tunable ASSUMPTIONS, not measured.")
        print(f"      gen tokens are a chars/{a.cpt} proxy for output tokens of the I/O layer only.")
        print(f"      This harness measures mechanics, not model reasoning quality.\n")

        report = {"params": vars(a), "rows": rows, "totals": {"0.6": T06, "0.7": T07},
                  "silent_error_free": {"0.6": f"{sf06}/{denom}", "0.7": f"{sf07}/{denom}"},
                  "writes_persisted": {"0.6": f"{pr06}/{denom}", "0.7": f"{pr07}/{denom}"}}
        (HERE / "results" / "latest.json").write_text(json.dumps(report, indent=2))
        print(f"wrote {HERE/'results'/'latest.json'}")
    finally:
        srv.terminate()

if __name__ == "__main__":
    main()
