ver 1.5.0 — six-improvement pass: retention, routing, self-firing memory

1. capture-update keeps the whole body (was truncating to first line)
2. frontmatter completeness: kind-default status + kind-seeded tags at
   capture, fm create-or-replace, incomplete-frontmatter lint, sweep
   backfill (one data source: KIND_STATUS/KIND_REQUIRED_FM)
3. pre-write duplicate gate (exit 76, --merge-into/--force)
4. recall corpus + ranking: sessions/journal indexed (down-weighted),
   BM25 x freshness x status fusion, recall --json, index schema 2
5. session hooks: SessionStart auto-load, Stop reflection nudge
6. one-tap triage verb with processing-log audit; --json on read verbs

+22 mock end-to-end tests; all suites green. Docs current (README,
CHANGELOG, SKILL.md, commands, references, MAINTENANCE, eval README).
Drops tracked .DS_Store (gitignored); retires README-gretchen.md
(moved to gitignored dist/); adds the field report that drove item 2.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

echo-memory.plugin.src/skills/echo-memory/scripts/echo_recall.py

@@ -20,15 +20,28 @@ DESIGN (bounded by the project's pure-stdlib, no-deps constraint):
   * Resilience     : if the index can't be built (vault unreachable), recall falls back
                      to the server's /search/simple so it degrades instead of dying.
 
-Corpus = the indexable ENTITY notes (idx_mod.kind_for_path is not None): people,
+Corpus = the indexable ENTITY notes (idx_mod.kind_for_path is not None) — people,
 companies, concepts, references, meetings, projects, areas, decisions, semantic/
-episodic memory, skills — the durable memory graph. (Sessions/journal are a future
-add; tracked in ROADMAP-1.0.md.)
+episodic memory, skills — PLUS the time-series notes where conversation-derived
+context actually lives: session logs (`_agent/sessions/`) and journal notes
+(`journal/daily|weekly|monthly|quarterly|annual/`). Time-series docs carry a
+per-kind DOWN-WEIGHT so entity notes still rank first for the same lexical match.
+
+RANKING (v1.5): final score = BM25 * kind_weight * freshness * status_factor —
+  * kind_weight : 1.0 entity · 0.5 session · 0.4 journal (KIND_WEIGHT)
+  * freshness   : half-life decay on `updated:` (or the filename date), floored at
+                  FRESH_FLOOR so old notes are demoted, never buried
+  * status      : `active` boosted, `archived` demoted (STATUS_FACTOR)
+so a stale archived project no longer outranks the live one on raw term frequency.
+Doc metadata rides in the index (schema 2); a schema-1 index is discarded and
+rebuilt on the next recall/sweep.
 """
 from __future__ import annotations
 
+import datetime as _dt
 import json
 import math
+import os
 import re
 import sys
 import urllib.parse
@@ -41,6 +54,7 @@ import echo_index as idx_mod     # noqa: E402
 import echo_links as links       # noqa: E402
 
 RECALL_INDEX_PATH = "_agent/index/recall-index.json"
+INDEX_SCHEMA = 2                 # bumped: schema 2 adds per-doc meta (weight/updated/status)
 
 # Tuning knobs. Defaults are standard BM25 (k1/b) + a 0.6 hop decay over <=2 hops.
 K1 = 1.5
@@ -49,6 +63,63 @@ MAX_HOPS = 2
 GRAPH_DECAY = 0.6
 MIN_SCORE = 0.0  # relevance floor; raise to trim weak lexical hits
 
+# --- rank-fusion priors -------------------------------------------------------
+# Time-series notes are in the corpus but down-weighted: they are where "what did we
+# decide three weeks ago" lives, yet the entity note should win a tie on the same terms.
+_TIME_SERIES = [
+    (re.compile(r"^_agent/sessions/"), "session"),
+    (re.compile(r"^journal/daily/"), "daily"),
+    (re.compile(r"^journal/(weekly|monthly|quarterly|annual)/"), "rollup"),
+]
+KIND_WEIGHT = {"session": 0.5, "daily": 0.4, "rollup": 0.4}
+STATUS_FACTOR = {"active": 1.1, "on-hold": 0.9, "archived": 0.75}
+FRESH_HALF_LIFE = float(os.environ.get("ECHO_FRESH_HALF_LIFE", "90"))  # days
+FRESH_FLOOR = 0.6   # freshness multiplier never drops below this — demote, don't bury
+
+_FM_UPDATED = re.compile(r"(?m)^updated:\s*[\"']?(\d{4}-\d{2}-\d{2})")
+_FM_STATUS = re.compile(r"(?m)^status:\s*[\"']?([A-Za-z-]+)")
+_PATH_DATE = re.compile(r"(\d{4}-\d{2}-\d{2})")
+
+
+def _series_kind(path: str) -> str | None:
+    for rx, kind in _TIME_SERIES:
+        if rx.match(path):
+            return kind
+    return None
+
+
+def doc_meta(path: str, raw_text: str) -> dict:
+    """Per-doc ranking priors, extracted once at index time: kind weight, last-updated
+    date (frontmatter `updated:`, else the date in the filename), and `status:`."""
+    kind = _series_kind(path)
+    w = KIND_WEIGHT.get(kind, 1.0) if kind else 1.0
+    head = raw_text[:2000]
+    mu = _FM_UPDATED.search(head)
+    updated = mu.group(1) if mu else None
+    if updated is None:
+        mp = _PATH_DATE.search(path.rsplit("/", 1)[-1])
+        updated = mp.group(1) if mp else None
+    ms = _FM_STATUS.search(head)
+    meta = {"w": w}
+    if updated:
+        meta["u"] = updated
+    if ms:
+        meta["s"] = ms.group(1)
+    return meta
+
+
+def freshness(updated: str | None, today_s: str) -> float:
+    """Half-life decay on document age, floored. Unknown age is neutral (1.0)."""
+    if not updated:
+        return 1.0
+    try:
+        age = (_dt.date.fromisoformat(today_s) - _dt.date.fromisoformat(updated)).days
+    except ValueError:
+        return 1.0
+    if age <= 0:
+        return 1.0
+    return FRESH_FLOOR + (1.0 - FRESH_FLOOR) * (0.5 ** (age / FRESH_HALF_LIFE))
+
 _WORD = re.compile(r"[a-z0-9]+")
 # Minimal English stoplist — keeps BM25 idf from being dominated by glue words.
 _STOP = frozenset(
@@ -73,12 +144,15 @@ def strip_frontmatter(text: str) -> str:
 
 # --------------------------------------------------------------------- BM25 core
 class Bm25Index:
-    """A compact, serializable BM25 index. add()/remove() are idempotent per path."""
+    """A compact, serializable BM25 index with per-doc ranking meta.
+    add()/remove() are idempotent per path. add() takes the RAW note text — it strips
+    frontmatter itself and extracts the doc meta (weight/updated/status) in one pass."""
 
     def __init__(self) -> None:
         self.df: Counter[str] = Counter()              # term -> #docs containing it
         self.postings: dict[str, dict[str, int]] = {}  # term -> {doc_path: tf}
         self.length: dict[str, int] = {}               # doc_path -> token count
+        self.meta: dict[str, dict] = {}                # doc_path -> {w, u, s} priors
         self.n_docs = 0
         self.avg_len = 0.0
 
@@ -86,11 +160,12 @@ class Bm25Index:
         self.n_docs = len(self.length)
         self.avg_len = (sum(self.length.values()) / self.n_docs) if self.n_docs else 0.0
 
-    def add(self, path: str, text: str) -> None:
+    def add(self, path: str, raw_text: str) -> None:
         if path in self.length:        # re-index in place (idempotent)
             self.remove(path)
-        toks = tokenize(text)
+        toks = tokenize(strip_frontmatter(raw_text))
         self.length[path] = len(toks)
+        self.meta[path] = doc_meta(path, raw_text)
         for term, tf in Counter(toks).items():
             self.postings.setdefault(term, {})[path] = tf
             self.df[term] += 1
@@ -107,9 +182,19 @@ class Bm25Index:
                     del self.postings[term]
                     del self.df[term]
         del self.length[path]
+        self.meta.pop(path, None)
         self._recompute()
 
-    def score(self, query: str, limit: int = 10) -> list[tuple[str, float]]:
+    def _doc_factor(self, path: str, today_s: str) -> float:
+        """The rank-fusion prior: kind weight x freshness x status factor."""
+        m = self.meta.get(path) or {}
+        return (float(m.get("w", 1.0))
+                * freshness(m.get("u"), today_s)
+                * STATUS_FACTOR.get(m.get("s", ""), 1.0))
+
+    def score(self, query: str, limit: int = 10,
+              today_s: str | None = None) -> list[tuple[str, float]]:
+        today_s = today_s or echo.today()
         scores: Counter[str] = Counter()
         for term in set(tokenize(query)):
             posting = self.postings.get(term)
@@ -120,18 +205,23 @@ class Bm25Index:
                 dl = self.length.get(path, 0)
                 denom = tf + K1 * (1 - B + B * (dl / self.avg_len if self.avg_len else 1))
                 scores[path] += idf * (tf * (K1 + 1) / denom if denom else 0)
-        return [(p, s) for p, s in scores.most_common(limit) if s > MIN_SCORE]
+        fused = {p: s * self._doc_factor(p, today_s) for p, s in scores.items()}
+        ranked = sorted(fused.items(), key=lambda kv: -kv[1])[:limit]
+        return [(p, s) for p, s in ranked if s > MIN_SCORE]
 
     # ---- serialization (compact; rebuildable, so the format can change freely) ----
     def to_json(self) -> dict:
-        return {"schema": 1, "n_docs": self.n_docs, "avg_len": self.avg_len,
-                "length": self.length, "postings": self.postings}
+        return {"schema": INDEX_SCHEMA, "n_docs": self.n_docs, "avg_len": self.avg_len,
+                "length": self.length, "postings": self.postings, "meta": self.meta}
 
     @classmethod
     def from_json(cls, d: dict) -> "Bm25Index":
+        if d.get("schema") != INDEX_SCHEMA:
+            return cls()  # older/newer format -> empty; recall's cold path rebuilds
         ix = cls()
         ix.length = d.get("length", {})
         ix.postings = d.get("postings", {})
+        ix.meta = d.get("meta", {})
         ix.n_docs = d.get("n_docs", len(ix.length))
         ix.avg_len = d.get("avg_len", 0.0)
         ix.df = Counter({term: len(posting) for term, posting in ix.postings.items()})
@@ -172,8 +262,12 @@ def _walk(prefix: str = ""):
 
 def _indexable(path: str) -> bool:
     base = path.rsplit("/", 1)[-1]
-    return (path.endswith(".md") and idx_mod.kind_for_path(path) is not None
-            and base not in _SKIP_BASENAMES and not _TEMPLATE_RE.search(path))
+    if (not path.endswith(".md") or base in _SKIP_BASENAMES
+            or _TEMPLATE_RE.search(path)):
+        return False
+    # Entity notes (the durable graph) + time-series notes (sessions, journal) — the
+    # latter join the corpus down-weighted so past decisions/discussions are findable.
+    return idx_mod.kind_for_path(path) is not None or _series_kind(path) is not None
 
 
 # ------------------------------------------------------------------- persistence
@@ -208,7 +302,7 @@ def rebuild(prefetched: dict | None = None) -> Bm25Index:
         items = ((p, _get(p)) for p in _walk() if _indexable(p))
     for path, text in items:
         if text is not None:
-            ix.add(path, strip_frontmatter(text))
+            ix.add(path, text)   # add() strips frontmatter + extracts doc meta itself
     save_index(ix)
     return ix
 
@@ -224,7 +318,7 @@ def update_note(path: str, text: str) -> None:
         import echo_concurrency
         with echo_concurrency.vault_lock():
             ix = load_index()                      # fresh read inside the lock
-            ix.add(path, strip_frontmatter(text))
+            ix.add(path, text)                     # raw text: add() strips + extracts meta
             save_index(ix)
     except Exception as exc:  # noqa: BLE001 — upkeep must never fail a capture
         print(f"echo_recall: index update skipped ({exc})", file=sys.stderr)
@@ -279,37 +373,57 @@ def expand_graph(seeds, nmap: dict, base_scores: dict, max_hops: int = MAX_HOPS)
 
 
 # ------------------------------------------------------------------- presentation
+def _doc_summary(path: str, text: str) -> dict:
+    """Structured per-note summary shared by the human brief and --json output."""
+    out: dict = {"path": path}
+    mtype = re.search(r"(?m)^type:\s*(.+)$", text)
+    if mtype:
+        out["type"] = mtype.group(1).strip()
+    mu = _FM_UPDATED.search(text[:2000])
+    if mu:
+        out["updated"] = mu.group(1)
+    ms = _FM_STATUS.search(text[:2000])
+    if ms:
+        out["status"] = ms.group(1)
+    body = strip_frontmatter(text)
+    status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body)
+    if status and status.group(1).strip():
+        out["excerpt"] = status.group(1).strip()[:400]
+    else:
+        kept = [ln[:200] for ln in body.splitlines()
+                if ln.strip() and not ln.startswith("# ")][:6]
+        out["excerpt"] = "\n".join(kept)
+    return out
+
+
 def _brief(path: str, score: float | None = None, via: str | None = None) -> None:
     text = links.get_text(path)
     if text is None:
         return
+    info = _doc_summary(path, text)
     head = f"\n### {path}"
     if score is not None:
         head += f"   (score {score:.2f})"
     if via:
         head += f"   (via {via})"
     print(head)
-    mtype = re.search(r"(?m)^type:\s*(.+)$", text)
-    if mtype:
-        print(f"_type: {mtype.group(1).strip()}_")
-    body = strip_frontmatter(text)
-    status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body)
-    if status and status.group(1).strip():
-        print(status.group(1).strip()[:400])
-        return
-    shown = 0
-    for line in body.splitlines():
-        if not line.strip() or line.startswith("# "):
-            continue
-        print(line[:200])
-        shown += 1
-        if shown >= 6:
-            break
+    stamps = []
+    if info.get("type"):
+        stamps.append(f"type: {info['type']}")
+    if info.get("updated"):
+        stamps.append(f"updated: {info['updated']}")
+    if info.get("status"):
+        stamps.append(f"status: {info['status']}")
+    if stamps:
+        print("_" + " · ".join(stamps) + "_")   # staleness is visible, not guessed
+    if info.get("excerpt"):
+        print(info["excerpt"])
 
 
 # ------------------------------------------------------------------- entrypoint
-def recall(query, limit: int = 8) -> int:
+def recall(query, limit: int = 8, as_json: bool = False) -> int:
     q = " ".join(query) if isinstance(query, list) else query
+    today_s = echo.today()
     index = idx_mod.load()
     nmap = idx_mod.name_map(index)
 
@@ -319,7 +433,7 @@ def recall(query, limit: int = 8) -> int:
         ix = load_index()
         if ix.n_docs == 0:
             ix = rebuild()
-        lexical = ix.score(q, limit=limit)
+        lexical = ix.score(q, limit=limit, today_s=today_s)
     except echo.EchoError as exc:
         print(f"recall: BM25 index unavailable ({exc}); falling back to server search",
               file=sys.stderr)
@@ -350,6 +464,26 @@ def recall(query, limit: int = 8) -> int:
     # --- graph layer ----------------------------------------------------------
     neighbours = expand_graph(hits, nmap, base, max_hops=MAX_HOPS)
 
+    if as_json:
+        import echo_output
+        texts = echo.read_many(hits + [p for p, _ in neighbours[: 2 * limit]])
+        primary = []
+        for p in hits:
+            if texts.get(p) is None:
+                continue
+            primary.append({**_doc_summary(p, texts[p]),
+                            "score": round(base.get(p, 0.0), 3)})
+        linked = []
+        for p, (sc, via) in neighbours[: 2 * limit]:
+            if texts.get(p) is None:
+                continue
+            linked.append({**_doc_summary(p, texts[p]),
+                           "score": round(sc, 3), "via": via})
+        env = echo_output.envelope("recall", {"query": q, "primary": primary,
+                                              "linked": linked})
+        print(json.dumps(env, ensure_ascii=False))
+        return 0
+
     print(f"== recall: {q} ==")
     print(f"\n# Primary hits ({len(hits)})")
     for p in hits: