This commit is contained in:
jason
2026-06-22 23:55:19 -05:00
parent 151b962662
commit d34fbf7626
15 changed files with 571 additions and 55 deletions
@@ -99,6 +99,12 @@ def main(argv: list[str] | None = None) -> int:
if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
and not TEMPLATE_RE.search(p)]
# Fetch every note's content ONCE, concurrently. All three passes below (index,
# recall, link symmetrize) read from this shared cache, so no file is fetched
# twice and link targets aren't re-fetched per reference.
texts = echo.read_many(all_files)
print(f"sweep: read {len(texts)} notes (concurrent, keep-alive)\n")
# ---- 2. (re)build entity index -------------------------------------------
index = idx_mod.load()
existing = dict(index.get("entities", {}))
@@ -110,10 +116,13 @@ def main(argv: list[str] | None = None) -> int:
continue
base = path.rsplit("/", 1)[-1][:-3]
slug = idx_mod.slugify(base)
text = get(path) or ""
text = texts.get(path) or ""
title = links.first_h1(text) or base
prev = existing.get(slug)
aliases = prev.get("aliases", []) if prev else []
# Frontmatter aliases are authoritative and re-folded every rebuild; prior index
# aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
prev_aliases = prev.get("aliases", []) if prev else []
aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
if not prev:
added += 1
@@ -126,7 +135,7 @@ def main(argv: list[str] | None = None) -> int:
# ---- 2b. (re)build the recall (BM25) index -------------------------------
if apply:
rix = echo_recall.rebuild()
rix = echo_recall.rebuild(prefetched=texts)
print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)")
else:
n_idx = sum(1 for p in all_files if echo_recall._indexable(p))
@@ -147,14 +156,14 @@ def main(argv: list[str] | None = None) -> int:
missing = set() # (target_path, source_path) — reciprocal link to add on target
for path in all_files:
text = get(path)
text = texts.get(path)
if text is None:
continue
for tgt in links.related_targets(text):
tp = resolve_target(tgt)
if not tp or tp == path:
continue
back = {resolve_target(t) for t in links.related_targets(get(tp) or "")}
back = {resolve_target(t) for t in links.related_targets(texts.get(tp) or "")}
if path not in back:
missing.add((tp, path))
missing = sorted(missing)