forked from jason/echo
ver 1.2
This commit is contained in:
@@ -14,11 +14,18 @@ existing one is no longer wrongly skipped), correct `::` heading-target handling
|
||||
frontmatter field patches, a read-back-confirmed advisory multi-writer lock, a
|
||||
one-call cold-start `load`, and scope show/set.
|
||||
|
||||
Network is keep-alive and connection-pooled (one persistent connection per thread,
|
||||
reused across requests), and `read_many` fans bulk GETs across a thread pool — so
|
||||
full-vault scripts (sweep, lint, recall rebuild) make a handful of warm round-trips
|
||||
instead of hundreds of fresh TLS handshakes, and no longer blow past tool timeouts.
|
||||
|
||||
Config (env overrides; defaults match the rest of the plugin):
|
||||
ECHO_BASE default https://echoapi.alwisp.com
|
||||
ECHO_KEY default the plugin bearer token (env overrides it)
|
||||
ECHO_VERIFY default 1 — read-back verify after a PUT
|
||||
ECHO_LOCK_TTL default 900 — seconds before an advisory lock is considered stale
|
||||
ECHO_TIMEOUT default 30 — per-request socket timeout (seconds)
|
||||
ECHO_WORKERS default 8 — concurrency for read_many bulk reads
|
||||
ECHO_TODAY YYYY-MM-DD to use as "today" (pass the conversation's currentDate)
|
||||
|
||||
Usage:
|
||||
@@ -46,15 +53,16 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import http.client
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
# Never let a non-ASCII byte (em-dash in our own output, or Unicode in a fetched
|
||||
@@ -76,6 +84,11 @@ DEFAULT_KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
|
||||
KEY = echo_secrets.resolve_key(DEFAULT_KEY)
|
||||
VERIFY = os.environ.get("ECHO_VERIFY", "1") != "0"
|
||||
LOCK_TTL = int(os.environ.get("ECHO_LOCK_TTL", "900"))
|
||||
# Per-request socket timeout. A single stuck file fails fast instead of stalling a
|
||||
# full-vault pass; with read_many concurrency it only blocks one worker, not the run.
|
||||
TIMEOUT = int(os.environ.get("ECHO_TIMEOUT", "30"))
|
||||
# Concurrency for bulk reads (read_many). I/O-bound, so threads bypass the GIL fine.
|
||||
MAX_WORKERS = max(1, int(os.environ.get("ECHO_WORKERS", "8")))
|
||||
|
||||
LOCK_PATH = "_agent/locks/vault.lock"
|
||||
|
||||
@@ -101,30 +114,105 @@ def vault_url(path: str) -> str:
|
||||
return f"{BASE}/vault/{safe}"
|
||||
|
||||
|
||||
# --- persistent connection pool (keep-alive) ---------------------------------
|
||||
# urllib.urlopen opened a fresh TCP+TLS connection per call; against a remote HTTPS
|
||||
# endpoint the repeated handshake dominated, so a full-vault pass made hundreds of
|
||||
# serial handshakes and blew past the tool/sandbox timeout. We keep ONE connection
|
||||
# per thread (http.client is not thread-safe, so each thread — main + every pool
|
||||
# worker — owns its own) and reuse it across requests. read_many fans out over a
|
||||
# thread pool, so the pool's connections stay warm for the duration of a sweep.
|
||||
_local = threading.local()
|
||||
MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _new_connection() -> http.client.HTTPConnection:
|
||||
parts = urllib.parse.urlsplit(BASE)
|
||||
host = parts.hostname or parts.path
|
||||
if parts.scheme == "http":
|
||||
return http.client.HTTPConnection(host, parts.port or 80, timeout=TIMEOUT)
|
||||
return http.client.HTTPSConnection(host, parts.port or 443, timeout=TIMEOUT)
|
||||
|
||||
|
||||
def _connection() -> http.client.HTTPConnection:
|
||||
conn = getattr(_local, "conn", None)
|
||||
if conn is None:
|
||||
conn = _new_connection()
|
||||
_local.conn = conn
|
||||
return conn
|
||||
|
||||
|
||||
def _drop_connection() -> None:
|
||||
conn = getattr(_local, "conn", None)
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
_local.conn = None
|
||||
|
||||
|
||||
def request(method: str, url: str, data: bytes | None = None,
|
||||
headers: dict[str, str] | None = None) -> tuple[int, bytes]:
|
||||
"""One bounded retry on transport failure (status 0) or 5xx. Returns (status, body)."""
|
||||
"""Issue one request over this thread's reused connection. Returns (status, body);
|
||||
status 0 == transport failure (echo_queue relies on this to trigger offline queueing).
|
||||
Retries: a stale pooled connection reconnects immediately; 5xx and other transport
|
||||
errors get one bounded sleep-retry, same as before."""
|
||||
merged = {"Authorization": f"Bearer {KEY}", **(headers or {})}
|
||||
parts = urllib.parse.urlsplit(url)
|
||||
target = parts.path or "/"
|
||||
if parts.query:
|
||||
target += "?" + parts.query
|
||||
last_status, last_body = 0, b""
|
||||
for attempt in range(2):
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=merged)
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.status, response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read()
|
||||
if exc.code >= 500 and attempt == 0:
|
||||
conn = _connection()
|
||||
conn.request(method, target, body=data, headers=merged)
|
||||
resp = conn.getresponse()
|
||||
body = resp.read() # drain fully so the connection can be reused
|
||||
status = resp.status
|
||||
if status >= 500 and attempt < MAX_ATTEMPTS - 1:
|
||||
_drop_connection()
|
||||
time.sleep(1)
|
||||
continue
|
||||
return exc.code, body
|
||||
except Exception as exc: # connection error, timeout, DNS, ...
|
||||
return status, body
|
||||
except (http.client.RemoteDisconnected, http.client.BadStatusLine,
|
||||
ConnectionResetError, BrokenPipeError) as exc:
|
||||
# The server closed an idle keep-alive connection between requests.
|
||||
# Expected, not a failure: reconnect and retry without backing off.
|
||||
_drop_connection()
|
||||
last_status, last_body = 0, str(exc).encode()
|
||||
if attempt == 0:
|
||||
continue
|
||||
except Exception as exc: # timeout, DNS, refused, TLS, ...
|
||||
_drop_connection()
|
||||
last_status, last_body = 0, str(exc).encode()
|
||||
if attempt < MAX_ATTEMPTS - 1:
|
||||
time.sleep(1)
|
||||
continue
|
||||
return last_status, last_body
|
||||
|
||||
|
||||
def get_text(path: str) -> str | None:
|
||||
"""GET a vault path -> decoded text, or None on 404/any error. Never raises, so a
|
||||
single unreadable note can't abort a bulk pass (resilience for sweep/lint)."""
|
||||
try:
|
||||
status, body = request("GET", vault_url(path))
|
||||
except Exception:
|
||||
return None
|
||||
return body.decode(errors="replace") if status == 200 else None
|
||||
|
||||
|
||||
def read_many(paths) -> dict[str, str | None]:
|
||||
"""Concurrently GET many vault paths -> {path: text_or_None}. The single biggest
|
||||
win for full-vault scripts: it turns hundreds of serial round-trips into MAX_WORKERS
|
||||
parallel ones over warm keep-alive connections. Resilient — a bad file maps to None."""
|
||||
unique = list(dict.fromkeys(paths))
|
||||
if not unique:
|
||||
return {}
|
||||
workers = min(MAX_WORKERS, len(unique))
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
return dict(zip(unique, pool.map(get_text, unique)))
|
||||
|
||||
|
||||
def check(status: int, body: bytes, context: str) -> None:
|
||||
if status == 404:
|
||||
raise EchoError(f"{context}: not found", 44)
|
||||
|
||||
@@ -26,6 +26,7 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import echo # noqa: E402
|
||||
import echo_quality # noqa: E402 — distinctive-token guards for fuzzy candidate matching
|
||||
|
||||
INDEX_PATH = "_agent/index/entities.json"
|
||||
|
||||
@@ -61,6 +62,48 @@ def slugify(text: str) -> str:
|
||||
return text[:40] or "untitled"
|
||||
|
||||
|
||||
def derive_aliases(title: str) -> list[str]:
|
||||
"""Safe, lossless name variants of a title — case/punctuation normalizations that are
|
||||
unambiguously the SAME name (hyphen<->space, lowercased). Deliberately NOT acronyms or
|
||||
token subsets: those collide across entities and would make resolve() match too much.
|
||||
Lets `echo-memory` and `echo memory` both resolve to a title written either way."""
|
||||
t = str(title or "").strip()
|
||||
if not t:
|
||||
return []
|
||||
low = t.lower()
|
||||
variants = {low, slugify(t), slugify(t).replace("-", " "),
|
||||
low.replace("-", " "), low.replace(" ", "-")}
|
||||
return sorted(v for v in variants if v)
|
||||
|
||||
|
||||
_WORD = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
||||
def _sig_tokens(s) -> set[str]:
|
||||
"""Distinctive tokens of a string: long enough and not a common word (reuses the
|
||||
auto-link guards), so 'data'/'api'/'the' can't drive a fuzzy match on their own."""
|
||||
return {t for t in _WORD.findall(str(s or "").lower())
|
||||
if len(t) >= echo_quality.MIN_NAME_LEN and t not in echo_quality._COMMON}
|
||||
|
||||
|
||||
def aliases_in_frontmatter(text: str) -> list[str]:
|
||||
"""Read an `aliases:` list (flow `[a, b]` or block `- a`) from a note's YAML frontmatter.
|
||||
Frontmatter is the durable, Obsidian-native home for aliases; sweep folds these into the
|
||||
index so they survive an index rebuild."""
|
||||
if not str(text).startswith("---"):
|
||||
return []
|
||||
end = text.find("\n---", 3)
|
||||
fm = text[:end] if end != -1 else text
|
||||
flow = re.search(r"(?m)^aliases:\s*\[(.*?)\]", fm)
|
||||
if flow:
|
||||
return [a.strip().strip('"').strip("'") for a in flow.group(1).split(",") if a.strip()]
|
||||
block = re.search(r"(?m)^aliases:\s*\n((?:[ \t]*-[ \t]*.+\n?)+)", fm)
|
||||
if block:
|
||||
return [ln.strip().lstrip("-").strip().strip('"').strip("'")
|
||||
for ln in block.group(1).splitlines() if ln.strip()]
|
||||
return []
|
||||
|
||||
|
||||
def derive_path(kind: str, slug: str, date: str | None = None, domain: str = "business") -> str:
|
||||
folder = KIND_FOLDER.get(kind)
|
||||
if folder is None:
|
||||
@@ -128,7 +171,9 @@ def _norm(s) -> str:
|
||||
|
||||
|
||||
def resolve(index: dict, mention: str):
|
||||
"""Return (slug, entry) for a mention matched by slug, title, or alias — else (None, None)."""
|
||||
"""Return (slug, entry) for a mention EXACTLY matched by slug, title, or alias — else
|
||||
(None, None). Exact-only by design: this drives capture's create-vs-update decision, so a
|
||||
loose match here would silently merge distinct entities. Fuzzy lookup is fuzzy_candidates()."""
|
||||
key = _norm(mention)
|
||||
ents = index.get("entities", {})
|
||||
if key in ents:
|
||||
@@ -143,6 +188,29 @@ def resolve(index: dict, mention: str):
|
||||
return None, None
|
||||
|
||||
|
||||
def fuzzy_candidates(index: dict, mention: str, limit: int = 5):
|
||||
"""Advisory near-matches for when resolve() finds no exact hit. Ranks entities sharing
|
||||
>=1 *distinctive* token (>= MIN_NAME_LEN, not a common word) with the mention, so a
|
||||
shortened or expanded name — 'echo memory' for the project 'echo' — surfaces the existing
|
||||
note instead of vanishing. Returns [(slug, entry, score)] sorted best-first. NEVER used to
|
||||
auto-merge (that's resolve's job); only to SURFACE 'did you mean' candidates."""
|
||||
mtoks = _sig_tokens(mention)
|
||||
if not mtoks:
|
||||
return []
|
||||
scored = []
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
etoks = set()
|
||||
for name in (slug, e.get("title", ""), *e.get("aliases", [])):
|
||||
etoks |= _sig_tokens(name)
|
||||
overlap = mtoks & etoks
|
||||
if not overlap or not etoks:
|
||||
continue
|
||||
score = round(len(overlap) / min(len(mtoks), len(etoks)), 3)
|
||||
scored.append((score, len(overlap), slug, e))
|
||||
scored.sort(key=lambda x: (-x[0], -x[1], x[2]))
|
||||
return [(slug, e, score) for score, _, slug, e in scored[:limit]]
|
||||
|
||||
|
||||
def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = None,
|
||||
aliases=None) -> dict:
|
||||
ents = index.setdefault("entities", {})
|
||||
@@ -150,7 +218,11 @@ def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = Non
|
||||
e["path"] = path
|
||||
e["kind"] = kind
|
||||
e["title"] = title or e.get("title") or slug
|
||||
merged = set(e.get("aliases", [])) | set(aliases or [])
|
||||
# Keep any prior + explicit aliases, and ALWAYS fold in the safe title variants so a
|
||||
# hyphen/space/case form of the name resolves even if no one passed --aliases. Drop any
|
||||
# alias that just equals the slug (resolve already checks the slug).
|
||||
merged = set(e.get("aliases", [])) | set(aliases or []) | set(derive_aliases(e["title"]))
|
||||
merged.discard(slug)
|
||||
e["aliases"] = sorted(a for a in merged if a)
|
||||
e["last_seen"] = echo.today()
|
||||
ents[slug] = e
|
||||
|
||||
@@ -36,11 +36,19 @@ def resolve(mention: str) -> int:
|
||||
slug, e = idx_mod.resolve(index, mention)
|
||||
if e:
|
||||
print(json.dumps({"match": True, "slug": slug, **e}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
# No exact match — surface fuzzy candidates so a shortened/expanded name (e.g. "echo
|
||||
# memory" for the project "echo") reveals the existing note instead of looking absent.
|
||||
cands = idx_mod.fuzzy_candidates(index, mention)
|
||||
out = {"match": False, "mention": mention, "suggest_slug": idx_mod.slugify(mention)}
|
||||
if cands:
|
||||
out["candidates"] = [{"slug": s, "path": c.get("path"), "title": c.get("title"),
|
||||
"kind": c.get("kind"), "score": sc} for s, c, sc in cands]
|
||||
out["note"] = ("no exact match, but similar entities exist (see candidates) — check "
|
||||
"these before creating a new note; reuse the right path or `echo.py link`.")
|
||||
else:
|
||||
print(json.dumps({"match": False, "mention": mention,
|
||||
"suggest_slug": idx_mod.slugify(mention),
|
||||
"note": "no index entry — derive a path with the right --kind and create it"},
|
||||
ensure_ascii=False, indent=2))
|
||||
out["note"] = "no index entry — derive a path with the right --kind and create it"
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
@@ -93,13 +101,16 @@ def ensure_daily_log(line: str) -> None:
|
||||
|
||||
# ----------------------------------------------------------------- capture ---
|
||||
def _build_note(fm_type: str, status_v: str, today_s: str, title: str,
|
||||
body_text: str, sources: list[str]) -> str:
|
||||
body_text: str, sources: list[str], aliases: list[str] | None = None) -> str:
|
||||
src = "[" + ", ".join(json.dumps(s) for s in sources) + "]"
|
||||
fm = ["---", f"type: {fm_type}"]
|
||||
if status_v:
|
||||
fm.append(f"status: {status_v}")
|
||||
fm += [f"created: {today_s}", f"updated: {today_s}", "tags: []",
|
||||
"agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""]
|
||||
fm += [f"created: {today_s}", f"updated: {today_s}", "tags: []"]
|
||||
if aliases:
|
||||
# Obsidian-native, durable home for aliases; sweep folds these back into the index.
|
||||
fm.append("aliases: [" + ", ".join(json.dumps(a) for a in aliases) + "]")
|
||||
fm += ["agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""]
|
||||
out = "\n".join(fm)
|
||||
if body_text.strip():
|
||||
out += body_text.strip() + "\n"
|
||||
@@ -141,11 +152,14 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
|
||||
# M4: in --json mode, swallow the helper "ok:" chatter so stdout is clean JSON.
|
||||
return contextlib.redirect_stdout(io.StringIO()) if as_json else contextlib.nullcontext()
|
||||
|
||||
def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False) -> int:
|
||||
def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False,
|
||||
near=None) -> int:
|
||||
if as_json:
|
||||
act = f"dry-run:{action}" if dry else action
|
||||
env = echo_output.envelope(act, {"kind": kind, "path": path, "title": title,
|
||||
"links_added": links}, ok=ok)
|
||||
data = {"kind": kind, "path": path, "title": title, "links_added": links}
|
||||
if near:
|
||||
data["near_duplicates"] = near
|
||||
env = echo_output.envelope(act, data, ok=ok)
|
||||
print(json.dumps(env, ensure_ascii=False), file=real_stdout)
|
||||
elif dry:
|
||||
print(f"would {action} {kind or '-'} -> {path}")
|
||||
@@ -170,29 +184,47 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
|
||||
|
||||
slug = idx_mod.slugify(title)
|
||||
index = idx_mod.load()
|
||||
_, existing = idx_mod.resolve(index, title)
|
||||
match_slug, existing = idx_mod.resolve(index, title)
|
||||
existing_reachable = bool(existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200)
|
||||
|
||||
if dry_run:
|
||||
if existing_reachable:
|
||||
return done("update", existing["path"], dry=True)
|
||||
s2 = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
|
||||
return done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain), dry=True)
|
||||
near = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
|
||||
return done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain),
|
||||
dry=True, near=near)
|
||||
|
||||
near_dupes: list[str] = []
|
||||
index_title = title # title to record in the index entry
|
||||
index_aliases = list(aliases)
|
||||
with quiet():
|
||||
if existing_reachable:
|
||||
# Reuse the matched entity's CANONICAL slug — never re-slug from the mention, or
|
||||
# two index slugs end up pointing at one note. Keep its title; learn the mention
|
||||
# as an alias when it's a new distinctive form, so this name resolves next time.
|
||||
slug = match_slug or slug
|
||||
path = existing["path"]
|
||||
kind = existing.get("kind", kind)
|
||||
index_title = existing.get("title") or title
|
||||
known = {idx_mod._norm(a) for a in existing.get("aliases", [])} | {match_slug}
|
||||
if idx_mod.slugify(title) not in known and len(title.strip()) >= echo_quality.MIN_NAME_LEN:
|
||||
index_aliases.append(title)
|
||||
_append_to_existing(path, kind, today_s, body_text)
|
||||
action = "updated"
|
||||
else:
|
||||
if not status_v and kind == "project":
|
||||
status_v = "active"
|
||||
# Surface (don't silently create alongside) an existing entity this resembles —
|
||||
# the duplication trap when a shortened name didn't resolve exactly.
|
||||
near_dupes = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
|
||||
# M2: don't let a 40-char-truncated slug silently collide with a different
|
||||
# entity already in the index — disambiguate to slug-2, slug-3, ...
|
||||
slug = echo_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
|
||||
path = idx_mod.derive_path(kind, slug, date=date, domain=domain)
|
||||
note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title, body_text, sources)
|
||||
note_aliases = sorted((set(index_aliases) | set(idx_mod.derive_aliases(title))) - {slug})
|
||||
note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title,
|
||||
body_text, sources, note_aliases)
|
||||
echo.cmd_put(path, echo.temp_file(note.encode()))
|
||||
action = "created"
|
||||
|
||||
@@ -201,7 +233,7 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
|
||||
# fresh-re-read transaction. Returns the fresh index (used by auto-link below).
|
||||
import echo_concurrency
|
||||
index = echo_concurrency.atomic_index_update(
|
||||
lambda idx: idx_mod.upsert(idx, slug, path, kind, title, aliases))
|
||||
lambda idx: idx_mod.upsert(idx, slug, path, kind, index_title, index_aliases))
|
||||
|
||||
# auto-link: any other known entity CONFIDENTLY named in the body (M2: is_confident_link
|
||||
# rejects short/common tokens so a 3-char alias or a word like "API" can't link everywhere).
|
||||
@@ -226,7 +258,12 @@ def capture(kind: str | None, title: str, file_arg: str | None, status_v: str =
|
||||
ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]")
|
||||
|
||||
if as_json:
|
||||
done(action, path, links=linked)
|
||||
done(action, path, links=linked, near=near_dupes)
|
||||
else:
|
||||
print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else ""))
|
||||
if near_dupes:
|
||||
kind_word = "entity" if len(near_dupes) == 1 else "entities"
|
||||
print(f"WARNING: similar existing {kind_word} ({', '.join(near_dupes)}) — if this is "
|
||||
f"the same thing, merge or `echo.py link` instead of keeping a duplicate.",
|
||||
file=real_stdout)
|
||||
return 0
|
||||
|
||||
@@ -195,14 +195,18 @@ def save_index(ix: Bm25Index) -> None:
|
||||
echo.check(status, b, "recall-index save")
|
||||
|
||||
|
||||
def rebuild() -> Bm25Index:
|
||||
def rebuild(prefetched: dict | None = None) -> Bm25Index:
|
||||
"""Full rebuild from every indexable entity note. Called by sweep.py and as the
|
||||
cold-cache path inside recall(). Saves and returns the index."""
|
||||
cold-cache path inside recall(). Saves and returns the index.
|
||||
|
||||
`prefetched` (a {path: text} map, e.g. from echo.read_many) lets sweep.py reuse the
|
||||
content it already pulled instead of walking and re-fetching the whole vault again."""
|
||||
ix = Bm25Index()
|
||||
for path in _walk():
|
||||
if not _indexable(path):
|
||||
continue
|
||||
text = _get(path)
|
||||
if prefetched is not None:
|
||||
items = ((p, t) for p, t in prefetched.items() if _indexable(p))
|
||||
else:
|
||||
items = ((p, _get(p)) for p in _walk() if _indexable(p))
|
||||
for path, text in items:
|
||||
if text is not None:
|
||||
ix.add(path, strip_frontmatter(text))
|
||||
save_index(ix)
|
||||
|
||||
@@ -99,6 +99,12 @@ def main(argv: list[str] | None = None) -> int:
|
||||
if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
|
||||
and not TEMPLATE_RE.search(p)]
|
||||
|
||||
# Fetch every note's content ONCE, concurrently. All three passes below (index,
|
||||
# recall, link symmetrize) read from this shared cache, so no file is fetched
|
||||
# twice and link targets aren't re-fetched per reference.
|
||||
texts = echo.read_many(all_files)
|
||||
print(f"sweep: read {len(texts)} notes (concurrent, keep-alive)\n")
|
||||
|
||||
# ---- 2. (re)build entity index -------------------------------------------
|
||||
index = idx_mod.load()
|
||||
existing = dict(index.get("entities", {}))
|
||||
@@ -110,10 +116,13 @@ def main(argv: list[str] | None = None) -> int:
|
||||
continue
|
||||
base = path.rsplit("/", 1)[-1][:-3]
|
||||
slug = idx_mod.slugify(base)
|
||||
text = get(path) or ""
|
||||
text = texts.get(path) or ""
|
||||
title = links.first_h1(text) or base
|
||||
prev = existing.get(slug)
|
||||
aliases = prev.get("aliases", []) if prev else []
|
||||
# Frontmatter aliases are authoritative and re-folded every rebuild; prior index
|
||||
# aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
|
||||
prev_aliases = prev.get("aliases", []) if prev else []
|
||||
aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
|
||||
idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
|
||||
if not prev:
|
||||
added += 1
|
||||
@@ -126,7 +135,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
# ---- 2b. (re)build the recall (BM25) index -------------------------------
|
||||
if apply:
|
||||
rix = echo_recall.rebuild()
|
||||
rix = echo_recall.rebuild(prefetched=texts)
|
||||
print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)")
|
||||
else:
|
||||
n_idx = sum(1 for p in all_files if echo_recall._indexable(p))
|
||||
@@ -147,14 +156,14 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
missing = set() # (target_path, source_path) — reciprocal link to add on target
|
||||
for path in all_files:
|
||||
text = get(path)
|
||||
text = texts.get(path)
|
||||
if text is None:
|
||||
continue
|
||||
for tgt in links.related_targets(text):
|
||||
tp = resolve_target(tgt)
|
||||
if not tp or tp == path:
|
||||
continue
|
||||
back = {resolve_target(t) for t in links.related_targets(get(tp) or "")}
|
||||
back = {resolve_target(t) for t in links.related_targets(texts.get(tp) or "")}
|
||||
if path not in back:
|
||||
missing.add((tp, path))
|
||||
missing = sorted(missing)
|
||||
|
||||
@@ -38,6 +38,29 @@ def test_frontmatter_existing_json_is_preserved() -> None:
|
||||
assert echo.normalize_json_scalar('"2026-06-20"') == b'"2026-06-20"'
|
||||
|
||||
|
||||
def test_read_many_dedups_and_maps_each_path() -> None:
|
||||
# Concurrency helper contract, no network: monkeypatch the per-file fetch.
|
||||
orig = echo.get_text
|
||||
echo.get_text = lambda p: f"T:{p}"
|
||||
try:
|
||||
assert echo.read_many(["a", "b", "a"]) == {"a": "T:a", "b": "T:b"}
|
||||
finally:
|
||||
echo.get_text = orig
|
||||
|
||||
|
||||
def test_read_many_empty_returns_empty_dict() -> None:
|
||||
assert echo.read_many([]) == {}
|
||||
|
||||
|
||||
def test_read_many_tolerates_unreadable_file_as_none() -> None:
|
||||
orig = echo.get_text
|
||||
echo.get_text = lambda p: None if p == "bad" else f"T:{p}"
|
||||
try:
|
||||
assert echo.read_many(["ok", "bad"]) == {"ok": "T:ok", "bad": None}
|
||||
finally:
|
||||
echo.get_text = orig
|
||||
|
||||
|
||||
def test_stem_extracts_literal_directory_prefix() -> None:
|
||||
assert check_routing.stem(r"^projects/active/[^/]+\.md$") == "projects/active/"
|
||||
assert check_routing.stem(r"^areas/(business|personal)/[^/]+\.md$") == "areas/"
|
||||
@@ -86,6 +109,44 @@ def test_kind_for_path() -> None:
|
||||
assert echo_index.kind_for_path("journal/daily/2026-01-01.md") is None
|
||||
|
||||
|
||||
def test_derive_aliases_produces_punctuation_variants() -> None:
|
||||
al = echo_index.derive_aliases("ECHO Memory")
|
||||
assert "echo-memory" in al and "echo memory" in al
|
||||
assert echo_index.derive_aliases("") == []
|
||||
|
||||
|
||||
def test_upsert_folds_in_title_variants_and_drops_slug() -> None:
|
||||
index = {"entities": {}}
|
||||
e = echo_index.upsert(index, "echo-memory", "projects/active/echo-memory.md",
|
||||
"project", "Echo Memory")
|
||||
assert "echo memory" in e["aliases"] # space variant auto-derived
|
||||
assert "echo-memory" not in e["aliases"] # equals the slug -> not stored redundantly
|
||||
|
||||
|
||||
def test_fuzzy_candidates_finds_shortened_name() -> None:
|
||||
# The real bug: a project slugged "echo" must surface for the mention "echo memory".
|
||||
index = {"entities": {"echo": {"path": "projects/active/echo.md", "kind": "project",
|
||||
"title": "echo", "aliases": []}}}
|
||||
cands = echo_index.fuzzy_candidates(index, "echo memory")
|
||||
assert cands and cands[0][0] == "echo"
|
||||
# exact resolve still misses (it's exact-only) — fuzzy is the safety net:
|
||||
assert echo_index.resolve(index, "echo memory")[0] is None
|
||||
|
||||
|
||||
def test_fuzzy_candidates_ignores_common_and_short_tokens() -> None:
|
||||
index = {"entities": {"data": {"path": "x.md", "kind": "concept",
|
||||
"title": "data", "aliases": []}}}
|
||||
assert echo_index.fuzzy_candidates(index, "data pipeline") == []
|
||||
|
||||
|
||||
def test_aliases_in_frontmatter_flow_and_block() -> None:
|
||||
flow = '---\ntype: project\naliases: ["echo memory", "echo plugin"]\n---\n# x\n'
|
||||
assert echo_index.aliases_in_frontmatter(flow) == ["echo memory", "echo plugin"]
|
||||
block = "---\ntype: project\naliases:\n - echo memory\n - echo plugin\n---\n# x\n"
|
||||
assert echo_index.aliases_in_frontmatter(block) == ["echo memory", "echo plugin"]
|
||||
assert echo_index.aliases_in_frontmatter("# no frontmatter\n") == []
|
||||
|
||||
|
||||
def test_links_add_related_is_idempotent_and_bidirectional_token() -> None:
|
||||
text = "---\ntype: person\n---\n\n# Bob\n\n## Related\n"
|
||||
new, changed = echo_links.add_related(text, "resources/companies/mpm.md")
|
||||
|
||||
@@ -143,6 +143,11 @@ def main() -> int:
|
||||
flag("routing-manifest", f"could not load routing.json ({exc}) — path checks skipped")
|
||||
|
||||
all_files = list(walk())
|
||||
md_files = [p for p in all_files if p.endswith(".md")]
|
||||
md_set = set(md_files)
|
||||
# Fetch every markdown note ONCE, concurrently; all per-note checks below read from
|
||||
# this shared cache instead of issuing a fresh GET per file per section.
|
||||
texts = echo.read_many(md_files)
|
||||
|
||||
# Path membership + retired-path detection
|
||||
for path in all_files:
|
||||
@@ -159,7 +164,7 @@ def main() -> int:
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
if base in SKIP or template_re.search(path) or not path.endswith(".md"):
|
||||
continue
|
||||
text = get(path) or ""
|
||||
text = texts.get(path) or ""
|
||||
raw, fm = parse_frontmatter(text)
|
||||
if "[[" in raw:
|
||||
flag("frontmatter-wikilink", f"{path}: '[[...]]' inside frontmatter")
|
||||
@@ -184,7 +189,7 @@ def main() -> int:
|
||||
continue
|
||||
slug = filename[:-3]
|
||||
slug_homes.setdefault(slug, []).append(lifecycle)
|
||||
text = get(f"projects/{lifecycle}/{filename}") or ""
|
||||
text = texts.get(f"projects/{lifecycle}/{filename}") or ""
|
||||
_, fm = parse_frontmatter(text)
|
||||
status = str(fm.get("status", "")).strip()
|
||||
if status and status != lifecycle:
|
||||
@@ -200,7 +205,7 @@ def main() -> int:
|
||||
# Daily notes: duplicate "## Agent Log" headings
|
||||
for path in all_files:
|
||||
if re.match(r"^journal/daily/.*\.md$", path):
|
||||
text = get(path) or ""
|
||||
text = texts.get(path) or ""
|
||||
count = len(re.findall(r"(?m)^## Agent Log\s*$", text))
|
||||
if count > 1:
|
||||
flag("duplicate-agent-log", f"{path}: {count} '## Agent Log' headings")
|
||||
@@ -232,8 +237,7 @@ def main() -> int:
|
||||
f"scope set {scope_updated}; {len(since)} session(s) logged since without a switch — confirm it still reflects current work (or run `echo.py scope set`)")
|
||||
|
||||
# ---- Graph health: broken wikilinks, orphans, index drift ----------------
|
||||
md_files = [p for p in all_files if p.endswith(".md")]
|
||||
md_set = set(md_files)
|
||||
# (md_files / md_set were built up top; reuse the shared `texts` cache.)
|
||||
basemap: dict[str, str] = {}
|
||||
for p in md_files:
|
||||
basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)
|
||||
@@ -248,13 +252,11 @@ def main() -> int:
|
||||
return cand if cand in md_set else None
|
||||
return basemap.get(idx_mod.slugify(target))
|
||||
|
||||
text_cache: dict[str, str] = {}
|
||||
inbound: set[str] = set()
|
||||
for path in md_files:
|
||||
if path.rsplit("/", 1)[-1] in SKIP or skip_link.search(path):
|
||||
continue
|
||||
text = get(path) or ""
|
||||
text_cache[path] = text
|
||||
text = texts.get(path) or ""
|
||||
_, body = split_frontmatter(text)
|
||||
for t in links.all_wikilinks(body):
|
||||
tp = resolve_link(t)
|
||||
@@ -273,7 +275,7 @@ def main() -> int:
|
||||
for path in md_files:
|
||||
if not is_indexable(path) or path in inbound:
|
||||
continue
|
||||
text = text_cache.get(path, get(path) or "")
|
||||
text = texts.get(path) or ""
|
||||
if not links.related_targets(text):
|
||||
flag("orphan", f"{path}: no inbound links and empty ## Related ({idx_mod.kind_for_path(path)}) — link it for recall")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user