forked from jason/echo
Phase 1: mechanical echo→chorus rename with back-compat shims
Identity rename, no behavior change (CHORUS-PLAN.md Phase 1): - Plugin echo-memory → chorus-memory: manifest (v2.0.0-alpha.1), skill dir, 16 scripts (chorus.py, chorus_config.py, …), EchoError → ChorusError, /chorus-* commands, hook paths, docs, scaffold seeds, eval harness, build.py. Docs endpoint → chorusapi.mpm.to. - Env ECHO_* → CHORUS_*; config → ~/.claude/chorus-memory/config.json; state dir → ~/.chorus-memory/; marker → _agent/chorus-vault.md. Back-compat shims (one major version): - chorus_config aliases legacy ECHO_* env at import; reads a legacy echo-memory config.json when no CHORUS config exists (writes never land there); doctor/config report the legacy source. - State dir honors an existing ~/.echo-memory when the new dir is absent (offline queue not stranded). - Marker dual-probe in load/doctor/bootstrap/lint/sweep/migrate: a pre-fork _agent/echo-vault.md counts as bootstrapped; bootstrap repair won't write a second marker; routing gains agent-marker-legacy (GET). Verified: 25/25 unit tests, scaffold + routing-sync checks, 4 mock e2e suites, run_eval metrics unchanged, +6 shim smoke tests green. Rebuilt chorus-memory.plugin (79 entries). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""bootstrap.py — stand up (or repair) an CHORUS vault deterministically.
|
||||
|
||||
Idempotent and additive: every write is probe-before-write and NEVER overwrites
|
||||
an existing file. The marker (_agent/chorus-vault.md) is written LAST, so the vault
|
||||
is only flagged "set up" once every piece is in place. Re-running is the repair
|
||||
path (it fills in only what is missing). Cross-platform: pure Python via chorus.py.
|
||||
|
||||
Usage:
|
||||
bootstrap.py [--dry-run]
|
||||
|
||||
Env: CHORUS_BASE, CHORUS_KEY (via chorus.py), CHORUS_TODAY (YYYY-MM-DD for {{DATE}}).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
SKILL_DIR = SCRIPT_DIR.parent
|
||||
SCAFFOLD = SKILL_DIR / "scaffold"
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import chorus # noqa: E402
|
||||
|
||||
LEAVES = [
|
||||
"inbox/captures", "inbox/imports", "inbox/processing-log",
|
||||
"journal/daily", "journal/weekly", "journal/monthly", "journal/quarterly",
|
||||
"journal/annual", "journal/templates",
|
||||
"projects/active", "projects/incubating", "projects/on-hold", "projects/archived",
|
||||
"areas/business", "areas/personal", "areas/learning", "areas/systems",
|
||||
"resources/concepts", "resources/references", "resources/people",
|
||||
"resources/companies", "resources/meetings",
|
||||
"decisions/by-date",
|
||||
"_agent/context", "_agent/memory/working", "_agent/memory/episodic",
|
||||
"_agent/memory/semantic", "_agent/sessions", "_agent/health", "_agent/templates",
|
||||
"_agent/heartbeat", "_agent/skills/active", "_agent/skills/archived", "_agent/locks",
|
||||
"_agent/index",
|
||||
]
|
||||
|
||||
|
||||
def exists(path: str) -> bool:
|
||||
status, _ = chorus.request("GET", chorus.vault_url(path))
|
||||
return status == 200
|
||||
|
||||
|
||||
def put_text(path: str, text: str) -> None:
|
||||
status, body = chorus.request("PUT", chorus.vault_url(path), data=text.encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
chorus.check(status, body, f"bootstrap put {path}")
|
||||
|
||||
|
||||
def seed(path: str, source: Path, dry_run: bool) -> None:
|
||||
if exists(path):
|
||||
print(f"bootstrap: skip (exists) {path}")
|
||||
return
|
||||
if dry_run:
|
||||
print(f"bootstrap: would seed {path} <- {source.relative_to(SKILL_DIR)}")
|
||||
return
|
||||
text = source.read_text(encoding="utf-8").replace("{{DATE}}", chorus.today())
|
||||
put_text(path, text)
|
||||
print(f"bootstrap: seeded {path}")
|
||||
|
||||
|
||||
def leaf_readme(folder: str, dry_run: bool) -> None:
|
||||
path = f"{folder}/README.md"
|
||||
if exists(path):
|
||||
return
|
||||
if dry_run:
|
||||
print(f"bootstrap: would readme {path}")
|
||||
return
|
||||
name = folder.rsplit("/", 1)[-1]
|
||||
put_text(path, f"# {name}\n\nMemory vault folder. See the chorus-memory plugin for conventions.\n")
|
||||
print(f"bootstrap: readme {path}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Bootstrap or repair an CHORUS vault")
|
||||
parser.add_argument("--dry-run", "-n", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not SCAFFOLD.is_dir():
|
||||
print(f"bootstrap: scaffold not found at {SCAFFOLD}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
legacy_marker = (not exists(chorus.MARKER_PATH)) and exists(chorus.LEGACY_MARKER_PATH)
|
||||
if exists(chorus.MARKER_PATH) or legacy_marker:
|
||||
probe = chorus.LEGACY_MARKER_PATH if legacy_marker else chorus.MARKER_PATH
|
||||
status, body = chorus.request("GET", chorus.vault_url(probe))
|
||||
version = "unknown"
|
||||
if status == 200:
|
||||
version = next((ln.split(":", 1)[1].strip()
|
||||
for ln in body.decode(errors="replace").splitlines()
|
||||
if ln.startswith("schema_version:")), "unknown")
|
||||
note = " [legacy ECHO marker]" if legacy_marker else ""
|
||||
print(f"bootstrap: marker present{note} (schema_version={version}). Running repair pass (fills only missing files).")
|
||||
|
||||
for folder in LEAVES:
|
||||
leaf_readme(folder, args.dry_run)
|
||||
|
||||
templates = SCAFFOLD / "templates"
|
||||
if templates.is_dir():
|
||||
for source in sorted(templates.rglob("*.md")):
|
||||
seed(source.relative_to(templates).as_posix(), source, args.dry_run)
|
||||
|
||||
seed("_agent/memory/semantic/operator-preferences.md", SCAFFOLD / "anchors" / "operator-preferences.seed.md", args.dry_run)
|
||||
seed("_agent/context/current-context.md", SCAFFOLD / "anchors" / "current-context.seed.md", args.dry_run)
|
||||
seed("inbox/captures/inbox.md", SCAFFOLD / "anchors" / "inbox.seed.md", args.dry_run)
|
||||
seed("README.md", SCAFFOLD / "README.vault.md", args.dry_run)
|
||||
if legacy_marker:
|
||||
# Adopted pre-fork ECHO vault: leave the legacy marker as the single
|
||||
# authoritative probe — the schema migration renames it. Two markers
|
||||
# would make "which is authoritative" ambiguous.
|
||||
print("bootstrap: legacy ECHO marker present — not writing the CHORUS marker (migrate.py renames it).")
|
||||
else:
|
||||
seed(chorus.MARKER_PATH, SCAFFOLD / "chorus-vault.md", args.dry_run) # marker LAST
|
||||
|
||||
print(f"bootstrap: done ({'DRY-RUN ' if args.dry_run else ''}{chorus.today()}).")
|
||||
print("bootstrap: next: create today's daily note + a bootstrap session log + heartbeat (see SKILL.md).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""check_routing.py — keep the routing VIEWS in sync with routing.json.
|
||||
|
||||
routing.json is the canonical route manifest. SKILL.md ("Where to Write"),
|
||||
references/routing-map.md, and references/api-reference.md are hand-maintained
|
||||
human views of it, guarded until now only by a prose "routing.json wins"
|
||||
convention. This check makes that convention mechanically true:
|
||||
|
||||
CHECK A (validity, all three docs): every concrete vault path mentioned in the
|
||||
docs must match some route OR retired pattern in routing.json. Catches
|
||||
a doc pointing at a path the manifest does not define (typo, stale row).
|
||||
|
||||
CHECK B (coverage, routing-map.md): every route in routing.json must be named
|
||||
by at least one path in routing-map.md, which claims to be the complete
|
||||
canonical map. Catches a route added to the manifest but never documented.
|
||||
|
||||
Exit: 0 = in sync, 1 = drift found, 2 = could not read inputs. No network, no vault.
|
||||
Run it in CI / from the eval harness; it is pure local file parsing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
SKILL_DIR = SCRIPT_DIR.parent
|
||||
ROUTING_JSON = SCRIPT_DIR / "routing.json"
|
||||
DOCS = {
|
||||
"SKILL.md": SKILL_DIR / "SKILL.md",
|
||||
"routing-map.md": SKILL_DIR / "references" / "routing-map.md",
|
||||
"api-reference.md": SKILL_DIR / "references" / "api-reference.md",
|
||||
}
|
||||
CANONICAL_MAP = "routing-map.md"
|
||||
|
||||
ROOTS = ("_agent", "inbox", "journal", "projects", "areas", "resources", "decisions")
|
||||
FILE_SUFFIXES = (".md", ".lock", ".json")
|
||||
|
||||
# Order matters: substitute the most specific placeholder first.
|
||||
DATE_SUBS = [
|
||||
("YYYY-MM-DD", "2026-01-02"),
|
||||
("YYYY-Www", "2026-W01"),
|
||||
("YYYY-MM", "2026-01"),
|
||||
("YYYY-Qn", "2026-Q1"),
|
||||
("HHMM", "0900"),
|
||||
("YYYY", "2026"),
|
||||
]
|
||||
|
||||
# Named placeholders whose route constrains the value to a fixed vocabulary, so a
|
||||
# generic "sample" would not match the pattern.
|
||||
NAMED_SUBS = {
|
||||
"<lifecycle>": "active",
|
||||
"<domain>": "business",
|
||||
}
|
||||
|
||||
|
||||
def load_routing():
|
||||
import json
|
||||
data = json.loads(ROUTING_JSON.read_text(encoding="utf-8"))
|
||||
routes = [(r["id"], re.compile(r["pattern"]), stem(r["pattern"])) for r in data.get("routes", [])]
|
||||
retired = [re.compile(r["pattern"]) for r in data.get("retired", [])]
|
||||
return routes, retired
|
||||
|
||||
|
||||
def stem(pattern: str) -> str:
|
||||
"""Literal directory prefix of a regex pattern, e.g. ^projects/active/[^/]+\\.md$ -> projects/active/."""
|
||||
body = pattern[1:] if pattern.startswith("^") else pattern
|
||||
literal = []
|
||||
for ch in body:
|
||||
if ch in "\\([{+*?.|^$":
|
||||
break
|
||||
literal.append(ch)
|
||||
text = "".join(literal)
|
||||
return text[: text.rfind("/") + 1] if "/" in text else ""
|
||||
|
||||
|
||||
def strip_fenced_blocks(text: str) -> str:
|
||||
out, in_fence = [], False
|
||||
for line in text.splitlines():
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if not in_fence:
|
||||
out.append(line)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def expand_braces(token: str) -> list[str]:
|
||||
"""Expand a single brace group: a/{x,y}/b -> [a/x/b, a/y/b]. Recurses for nested groups."""
|
||||
match = re.search(r"\{([^{}]*)\}", token)
|
||||
if not match:
|
||||
return [token]
|
||||
pre, post = token[: match.start()], token[match.end():]
|
||||
results = []
|
||||
for part in match.group(1).split(","):
|
||||
results.extend(expand_braces(pre + part.strip() + post))
|
||||
return results
|
||||
|
||||
|
||||
def concretize(token: str) -> str:
|
||||
for placeholder, sample in DATE_SUBS:
|
||||
token = token.replace(placeholder, sample)
|
||||
for placeholder, sample in NAMED_SUBS.items():
|
||||
token = token.replace(placeholder, sample)
|
||||
return re.sub(r"<[^>]+>", "sample", token)
|
||||
|
||||
|
||||
def looks_like_path(token: str) -> bool:
|
||||
if " " in token or "::" in token or token.count("`"):
|
||||
return False
|
||||
# README signposts can live under any folder, including placeholder folders.
|
||||
if token == "README.md" or token.endswith("/README.md"):
|
||||
return True
|
||||
# Otherwise must sit under a known vault root directory (root + "/"), which
|
||||
# excludes shorthand like `inbox.md` and non-paths like `application/vnd...`.
|
||||
if not any(token.startswith(root + "/") for root in ROOTS):
|
||||
return False
|
||||
return token.endswith("/") or token.endswith(FILE_SUFFIXES) or "<" in token or "{" in token
|
||||
|
||||
|
||||
def extract_tokens(text: str) -> set[str]:
|
||||
text = strip_fenced_blocks(text)
|
||||
tokens: set[str] = set()
|
||||
for raw in re.findall(r"`([^`]+)`", text):
|
||||
raw = raw.strip()
|
||||
for piece in expand_braces(raw):
|
||||
if looks_like_path(piece):
|
||||
tokens.add(piece)
|
||||
return tokens
|
||||
|
||||
|
||||
def classify(tokens: set[str]):
|
||||
files, dirs = set(), set()
|
||||
for token in tokens:
|
||||
if token.endswith("/"):
|
||||
dirs.add(token)
|
||||
else:
|
||||
files.add(concretize(token))
|
||||
return files, dirs
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
routes, retired = load_routing()
|
||||
except Exception as exc:
|
||||
print(f"check-routing: cannot read routing.json ({exc})", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
problems: list[str] = []
|
||||
doc_tokens: dict[str, tuple[set[str], set[str]]] = {}
|
||||
for name, path in DOCS.items():
|
||||
try:
|
||||
files, dirs = classify(extract_tokens(path.read_text(encoding="utf-8")))
|
||||
except Exception as exc:
|
||||
print(f"check-routing: cannot read {name} ({exc})", file=sys.stderr)
|
||||
return 2
|
||||
doc_tokens[name] = (files, dirs)
|
||||
|
||||
route_stems = {st for _, _, st in routes if st}
|
||||
|
||||
# CHECK A — every documented path is valid under the manifest.
|
||||
for name, (files, dirs) in doc_tokens.items():
|
||||
for f in sorted(files):
|
||||
if not (any(rx.match(f) for _, rx, _ in routes) or any(rx.match(f) for rx in retired)):
|
||||
problems.append(f"[A] {name}: path `{f}` matches no route or retired pattern in routing.json")
|
||||
for d in sorted(dirs):
|
||||
ok = (any(rx.match(d) for rx in retired)
|
||||
or any(st == d or st.startswith(d) or d.startswith(st) for st in route_stems))
|
||||
if not ok:
|
||||
problems.append(f"[A] {name}: folder `{d}` matches no route stem or retired pattern")
|
||||
|
||||
# CHECK B — every route appears in the canonical map.
|
||||
map_files, map_dirs = doc_tokens[CANONICAL_MAP]
|
||||
for rid, rx, st in routes:
|
||||
covered = any(rx.match(f) for f in map_files) or (st and st in map_dirs)
|
||||
if not covered:
|
||||
problems.append(f"[B] route '{rid}' ({rx.pattern}) is not documented in {CANONICAL_MAP}")
|
||||
|
||||
if problems:
|
||||
print(f"check-routing: {len(problems)} drift issue(s) found\n")
|
||||
for p in problems:
|
||||
print(f" - {p}")
|
||||
return 1
|
||||
print(f"check-routing: in sync — {len(routes)} routes documented and all doc paths are valid.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,910 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus.py — the single validated, cross-platform client for the CHORUS Obsidian
|
||||
Local REST API.
|
||||
|
||||
Replaces the original chorus.sh so the toolchain runs identically on Windows,
|
||||
macOS, and Linux with no bash / GNU-vs-BSD `date` dependency — only a Python 3
|
||||
interpreter (which the linter already requires).
|
||||
|
||||
Every read/write to the vault should go through this script. It centralizes:
|
||||
auth injection, HTTP-status checking (non-zero exit on >=400 so a failed write
|
||||
can never look like success), one bounded retry on 5xx/connection errors,
|
||||
WHOLE-LINE idempotent append (a new line that is merely a substring of an
|
||||
existing one is no longer wrongly skipped), correct `::` heading-target handling,
|
||||
frontmatter field patches, a read-back-confirmed advisory multi-writer lock, a
|
||||
one-call cold-start `load`, and scope show/set.
|
||||
|
||||
Network is keep-alive and connection-pooled (one persistent connection per thread,
|
||||
reused across requests), and `read_many` fans bulk GETs across a thread pool — so
|
||||
full-vault scripts (sweep, lint, recall rebuild) make a handful of warm round-trips
|
||||
instead of hundreds of fresh TLS handshakes, and no longer blow past tool timeouts.
|
||||
|
||||
Config (owner/endpoint/key are machine-local — see chorus_config; the plugin ships none):
|
||||
owner/endpoint/key resolved by chorus_config from ~/.claude/chorus-memory/config.json
|
||||
(env CHORUS_OWNER / CHORUS_BASE / CHORUS_KEY override per field).
|
||||
Set up a machine with `chorus.py config init` then edit, or
|
||||
`chorus.py config set --owner ... --endpoint ... --key ...`.
|
||||
CHORUS_VERIFY default 1 — read-back verify after a PUT
|
||||
CHORUS_LOCK_TTL default 900 — seconds before an advisory lock is considered stale
|
||||
CHORUS_TIMEOUT default 30 — per-request socket timeout (seconds)
|
||||
CHORUS_WORKERS default 8 — concurrency for read_many bulk reads
|
||||
CHORUS_TODAY YYYY-MM-DD to use as "today" (pass the conversation's currentDate)
|
||||
|
||||
Usage:
|
||||
chorus.py load # cold-start: the 6 orientation reads in one call
|
||||
chorus.py get <path> # print file contents (404 -> exit 44)
|
||||
chorus.py map <path> # document-map JSON (headings/blocks/frontmatter)
|
||||
chorus.py ls <dir> # directory listing JSON
|
||||
chorus.py search <query...> # /search/simple
|
||||
chorus.py put <path> [file] # create/overwrite (body from file or stdin)
|
||||
chorus.py post <path> [file] # raw append (NON-idempotent; prefer `append`)
|
||||
chorus.py append <path> <line> # idempotent append: skips only on an exact whole-line match
|
||||
chorus.py patch <path> <append|prepend|replace> <heading|frontmatter|block> <target> [file]
|
||||
chorus.py fm <path> <field> <json-value> # create-or-replace a frontmatter scalar
|
||||
chorus.py bump <path> [YYYY-MM-DD] # set frontmatter updated: to today (or given date)
|
||||
chorus.py triage [--list --json | <proposals.json> [--apply]] # one-tap inbox triage + audit log
|
||||
chorus.py delete <path> # DELETE (destructive; explicit use only)
|
||||
chorus.py lock <owner-id> # acquire advisory lock (exit 75 if held & fresh, or lost the race)
|
||||
chorus.py unlock <owner-id> # release advisory lock if owned by <owner-id>
|
||||
chorus.py scope show # print active scope, its freshness, and sessions-since
|
||||
chorus.py scope set "<text>" # switch scope atomically (history + replace + stamp scope_updated)
|
||||
chorus.py config [show|init|set|import] # machine-local owner/endpoint/key (config import <file> adopts a provided file)
|
||||
|
||||
Exit codes: 0 ok · 44 not-found(404) · 75 lock-held/lost · 76 duplicate-gate (capture) ·
|
||||
78 config-required · 2 usage · 1 other HTTP/transport error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import http.client
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
# Never let a non-ASCII byte (em-dash in our own output, or Unicode in a fetched
|
||||
# note) raise UnicodeEncodeError on a legacy Windows console.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Owner / endpoint / key are machine-local and resolved by chorus_config from
|
||||
# ~/.claude/chorus-memory/config.json (env CHORUS_OWNER / CHORUS_BASE / CHORUS_KEY override).
|
||||
# The plugin ships none of them; load() returns "" for anything unset so --help,
|
||||
# `config`, and `doctor` stay usable. Network ops raise a clear error if endpoint/key
|
||||
# are missing (see _require_endpoint / request).
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus_config # noqa: E402
|
||||
_CFG = chorus_config.load()
|
||||
BASE = _CFG["endpoint"]
|
||||
KEY = _CFG["key"]
|
||||
OWNER = _CFG["owner"]
|
||||
VERIFY = os.environ.get("CHORUS_VERIFY", "1") != "0"
|
||||
LOCK_TTL = int(os.environ.get("CHORUS_LOCK_TTL", "900"))
|
||||
# Per-request socket timeout. A single stuck file fails fast instead of stalling a
|
||||
# full-vault pass; with read_many concurrency it only blocks one worker, not the run.
|
||||
TIMEOUT = int(os.environ.get("CHORUS_TIMEOUT", "30"))
|
||||
# Concurrency for bulk reads (read_many). I/O-bound, so threads bypass the GIL fine.
|
||||
MAX_WORKERS = max(1, int(os.environ.get("CHORUS_WORKERS", "8")))
|
||||
|
||||
LOCK_PATH = "_agent/locks/vault.lock"
|
||||
|
||||
MARKER_PATH = "_agent/chorus-vault.md"
|
||||
# Pre-fork ECHO vaults carry the old marker name. It is honored read-only as a
|
||||
# bootstrap probe (vault-adoption path); the schema migration renames it.
|
||||
LEGACY_MARKER_PATH = "_agent/echo-vault.md"
|
||||
|
||||
|
||||
def marker_get():
|
||||
"""GET the bootstrap marker: the CHORUS name first, then the legacy ECHO
|
||||
name. Returns (path, status, body) — on a double miss, the CHORUS path with
|
||||
its 404 (or error) result."""
|
||||
status, body = request("GET", vault_url(MARKER_PATH))
|
||||
if status == 404:
|
||||
l_status, l_body = request("GET", vault_url(LEGACY_MARKER_PATH))
|
||||
if l_status == 200:
|
||||
return LEGACY_MARKER_PATH, l_status, l_body
|
||||
return MARKER_PATH, status, body
|
||||
|
||||
|
||||
class ChorusError(RuntimeError):
|
||||
def __init__(self, message: str, code: int = 1) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
def today() -> str:
|
||||
return os.environ.get("CHORUS_TODAY") or dt.date.today().isoformat()
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def vault_url(path: str) -> str:
|
||||
safe = "/".join(urllib.parse.quote(part) for part in path.split("/"))
|
||||
if path.endswith("/") and not safe.endswith("/"):
|
||||
safe += "/"
|
||||
return f"{BASE}/vault/{safe}"
|
||||
|
||||
|
||||
# --- persistent connection pool (keep-alive) ---------------------------------
|
||||
# urllib.urlopen opened a fresh TCP+TLS connection per call; against a remote HTTPS
|
||||
# endpoint the repeated handshake dominated, so a full-vault pass made hundreds of
|
||||
# serial handshakes and blew past the tool/sandbox timeout. We keep ONE connection
|
||||
# per thread (http.client is not thread-safe, so each thread — main + every pool
|
||||
# worker — owns its own) and reuse it across requests. read_many fans out over a
|
||||
# thread pool, so the pool's connections stay warm for the duration of a sweep.
|
||||
_local = threading.local()
|
||||
MAX_ATTEMPTS = 3
|
||||
|
||||
|
||||
def _require_config() -> None:
|
||||
"""Fail loudly (not with an opaque DNS/connection error) when this machine has no
|
||||
usable key file yet — the most common fresh-install state. Also catches an
|
||||
untouched `config init` template (placeholder endpoint/key)."""
|
||||
if not chorus_config.is_configured(_CFG):
|
||||
raise ChorusError(
|
||||
"chorus: NOT CONFIGURED — no usable CHORUS key file on this machine. Ask the "
|
||||
f"operator for their config (owner/endpoint/key) and install it at "
|
||||
f"{chorus_config.config_path()}: `chorus.py config import <file>`, or "
|
||||
"`chorus.py config set --owner ... --endpoint ... --key ...` "
|
||||
"(or set CHORUS_BASE / CHORUS_KEY).", 2)
|
||||
|
||||
|
||||
def _new_connection() -> http.client.HTTPConnection:
|
||||
parts = urllib.parse.urlsplit(BASE)
|
||||
host = parts.hostname or parts.path
|
||||
if parts.scheme == "http":
|
||||
return http.client.HTTPConnection(host, parts.port or 80, timeout=TIMEOUT)
|
||||
return http.client.HTTPSConnection(host, parts.port or 443, timeout=TIMEOUT)
|
||||
|
||||
|
||||
def _connection() -> http.client.HTTPConnection:
|
||||
conn = getattr(_local, "conn", None)
|
||||
if conn is None:
|
||||
conn = _new_connection()
|
||||
_local.conn = conn
|
||||
return conn
|
||||
|
||||
|
||||
def _drop_connection() -> None:
|
||||
conn = getattr(_local, "conn", None)
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
_local.conn = None
|
||||
|
||||
|
||||
def request(method: str, url: str, data: bytes | None = None,
|
||||
headers: dict[str, str] | None = None) -> tuple[int, bytes]:
|
||||
"""Issue one request over this thread's reused connection. Returns (status, body);
|
||||
status 0 == transport failure (chorus_queue relies on this to trigger offline queueing).
|
||||
Retries: a stale pooled connection reconnects immediately; 5xx and other transport
|
||||
errors get one bounded sleep-retry, same as before."""
|
||||
_require_config()
|
||||
merged = {"Authorization": f"Bearer {KEY}", **(headers or {})}
|
||||
parts = urllib.parse.urlsplit(url)
|
||||
target = parts.path or "/"
|
||||
if parts.query:
|
||||
target += "?" + parts.query
|
||||
last_status, last_body = 0, b""
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
try:
|
||||
conn = _connection()
|
||||
conn.request(method, target, body=data, headers=merged)
|
||||
resp = conn.getresponse()
|
||||
body = resp.read() # drain fully so the connection can be reused
|
||||
status = resp.status
|
||||
if status >= 500 and attempt < MAX_ATTEMPTS - 1:
|
||||
_drop_connection()
|
||||
time.sleep(1)
|
||||
continue
|
||||
return status, body
|
||||
except (http.client.RemoteDisconnected, http.client.BadStatusLine,
|
||||
ConnectionResetError, BrokenPipeError) as exc:
|
||||
# The server closed an idle keep-alive connection between requests.
|
||||
# Expected, not a failure: reconnect and retry without backing off.
|
||||
_drop_connection()
|
||||
last_status, last_body = 0, str(exc).encode()
|
||||
continue
|
||||
except Exception as exc: # timeout, DNS, refused, TLS, ...
|
||||
_drop_connection()
|
||||
last_status, last_body = 0, str(exc).encode()
|
||||
if attempt < MAX_ATTEMPTS - 1:
|
||||
time.sleep(1)
|
||||
continue
|
||||
return last_status, last_body
|
||||
|
||||
|
||||
def get_text(path: str) -> str | None:
|
||||
"""GET a vault path -> decoded text, or None on 404/any error. Never raises, so a
|
||||
single unreadable note can't abort a bulk pass (resilience for sweep/lint)."""
|
||||
try:
|
||||
status, body = request("GET", vault_url(path))
|
||||
except Exception:
|
||||
return None
|
||||
return body.decode(errors="replace") if status == 200 else None
|
||||
|
||||
|
||||
def read_many(paths) -> dict[str, str | None]:
|
||||
"""Concurrently GET many vault paths -> {path: text_or_None}. The single biggest
|
||||
win for full-vault scripts: it turns hundreds of serial round-trips into MAX_WORKERS
|
||||
parallel ones over warm keep-alive connections. Resilient — a bad file maps to None."""
|
||||
unique = list(dict.fromkeys(paths))
|
||||
if not unique:
|
||||
return {}
|
||||
workers = min(MAX_WORKERS, len(unique))
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
return dict(zip(unique, pool.map(get_text, unique)))
|
||||
|
||||
|
||||
def check(status: int, body: bytes, context: str) -> None:
|
||||
if status == 404:
|
||||
raise ChorusError(f"{context}: not found", 44)
|
||||
if status == 0:
|
||||
raise ChorusError(f"{context}: vault unreachable ({body.decode(errors='replace')}) [{BASE}]", 1)
|
||||
if status >= 400:
|
||||
raise ChorusError(f"{context}: HTTP {status} - {body.decode(errors='replace')}", 1)
|
||||
|
||||
|
||||
def read_body(arg: str | None) -> bytes:
|
||||
if arg in (None, "-"):
|
||||
raw = sys.stdin.buffer.read()
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode(locale.getpreferredencoding(False) or "cp1252", errors="replace")
|
||||
return text.encode("utf-8")
|
||||
return Path(arg).read_bytes()
|
||||
|
||||
|
||||
def normalize_patch_body(data: bytes, op: str, target_type: str) -> bytes:
|
||||
"""Heading PATCH bodies need a leading newline on replace and a trailing
|
||||
newline on append/prepend so the API does not concatenate onto adjacent text."""
|
||||
if target_type != "heading":
|
||||
return data
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
if op == "replace":
|
||||
text = text.strip("\r\n")
|
||||
text = f"\n{text}\n" if text else "\n"
|
||||
elif text and not text.endswith("\n"):
|
||||
text += "\n"
|
||||
return text.encode("utf-8")
|
||||
|
||||
|
||||
def normalize_json_scalar(value: str) -> bytes:
|
||||
"""Accept either a raw scalar ('Fabrication Lead') or pre-formed JSON ('"2026-06-20"')."""
|
||||
value = value.strip()
|
||||
try:
|
||||
json.loads(value)
|
||||
return value.encode("utf-8")
|
||||
except json.JSONDecodeError:
|
||||
return json.dumps(value).encode("utf-8")
|
||||
|
||||
|
||||
def temp_file(data: bytes) -> str:
|
||||
fh = tempfile.NamedTemporaryFile(delete=False)
|
||||
try:
|
||||
fh.write(data)
|
||||
return fh.name
|
||||
finally:
|
||||
fh.close()
|
||||
|
||||
|
||||
# ---- commands ----------------------------------------------------------------
|
||||
|
||||
def cmd_get(path: str) -> int:
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"get {path}")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_map(path: str) -> int:
|
||||
status, body = request("GET", vault_url(path),
|
||||
headers={"Accept": "application/vnd.olrapi.document-map+json"})
|
||||
check(status, body, f"map {path}")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_ls(path: str) -> int:
|
||||
if not path.endswith("/"):
|
||||
path += "/"
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"ls {path}")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_search(query: list[str]) -> int:
|
||||
q = urllib.parse.quote_plus(" ".join(query))
|
||||
status, body = request("POST", f"{BASE}/search/simple/?query={q}")
|
||||
check(status, body, "search")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def _qwrite(method: str, url: str, data: bytes | None = None, headers: dict | None = None):
|
||||
"""A mutating request that survives an outage: routes through chorus_queue.safe_request,
|
||||
which enqueues the write and returns (202, ...) when the vault is unreachable (H2)."""
|
||||
import chorus_queue
|
||||
return chorus_queue.safe_request(method, url, data=data, headers=headers)
|
||||
|
||||
|
||||
def cmd_put(path: str, file_arg: str | None) -> int:
|
||||
data = read_body(file_arg)
|
||||
status, body = _qwrite("PUT", vault_url(path), data=data,
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
if status == 202:
|
||||
print(f"queued (offline): PUT {path} — will sync on the next reachable session")
|
||||
return 0
|
||||
check(status, body, f"put {path}")
|
||||
if VERIFY:
|
||||
status, _ = request("GET", vault_url(path))
|
||||
if status != 200:
|
||||
raise ChorusError(f"put {path}: write did not verify (GET returned {status})")
|
||||
print(f"ok: PUT {path}")
|
||||
# Keep the recall (BM25) index current when a corpus note is written directly —
|
||||
# session logs, journal notes, hand-authored entity notes. update_note early-returns
|
||||
# for non-corpus paths and never raises, so this can't fail or slow a normal put.
|
||||
try:
|
||||
import chorus_recall
|
||||
chorus_recall.update_note(path, data.decode("utf-8", errors="replace"))
|
||||
except Exception as exc: # noqa: BLE001 — index upkeep must never fail a put
|
||||
print(f"chorus.py: recall-index update skipped ({exc})", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_post(path: str, file_arg: str | None) -> int:
|
||||
data = read_body(file_arg)
|
||||
status, body = _qwrite("POST", vault_url(path), data=data,
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
if status == 202:
|
||||
print(f"queued (offline): POST {path} — will sync on the next reachable session")
|
||||
return 0
|
||||
check(status, body, f"post {path}")
|
||||
print(f"ok: POST {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_append(path: str, line: str) -> int:
|
||||
"""Idempotent append. Skips ONLY when the exact whole line already exists —
|
||||
a new line that is a substring of an existing one is still appended."""
|
||||
status, body = request("GET", vault_url(path))
|
||||
if status == 200:
|
||||
existing = [ln.rstrip("\r") for ln in body.decode(errors="replace").splitlines()]
|
||||
if line.rstrip("\r") in existing:
|
||||
print(f"skip: line already present in {path}")
|
||||
return 0
|
||||
elif status == 0:
|
||||
pass # offline: skip the idempotency read; queue the POST (flush dedups by content)
|
||||
elif status != 404:
|
||||
check(status, body, f"append(read) {path}")
|
||||
status, body = _qwrite("POST", vault_url(path), data=f"{line}\n".encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
if status == 202:
|
||||
print(f"queued (offline): APPEND {path} — will sync on the next reachable session")
|
||||
return 0
|
||||
check(status, body, f"append {path}")
|
||||
print(f"ok: APPEND {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_patch(path: str, op: str, target_type: str, target: str, file_arg: str | None) -> int:
|
||||
if op not in {"append", "prepend", "replace"}:
|
||||
raise ChorusError("patch: op must be append, prepend, or replace", 2)
|
||||
if target_type not in {"heading", "frontmatter", "block"}:
|
||||
raise ChorusError("patch: target-type must be heading, frontmatter, or block", 2)
|
||||
data = normalize_patch_body(read_body(file_arg), op, target_type)
|
||||
status, body = _qwrite(
|
||||
"PATCH",
|
||||
vault_url(path),
|
||||
data=data,
|
||||
headers={
|
||||
"Operation": op,
|
||||
"Target-Type": target_type,
|
||||
"Target": target,
|
||||
"Content-Type": "application/json" if target_type == "frontmatter" else "text/markdown",
|
||||
},
|
||||
)
|
||||
if status == 202:
|
||||
print(f"queued (offline): PATCH {op} '{target}' -> {path} — will sync on the next reachable session")
|
||||
return 0
|
||||
check(status, body, f"patch {path} ({target})")
|
||||
print(f"ok: PATCH {op} {target_type} '{target}' -> {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def _yaml_flow(value) -> str:
|
||||
"""Render a JSON-decoded value as a YAML flow scalar/list for one frontmatter line."""
|
||||
if isinstance(value, list):
|
||||
return "[" + ", ".join(_yaml_flow(v) for v in value) + "]"
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if value is None:
|
||||
return ""
|
||||
s = str(value)
|
||||
return s if re.match(r"^[A-Za-z0-9][A-Za-z0-9 ._/-]*$", s) else json.dumps(s)
|
||||
|
||||
|
||||
def _fm_upsert_line(text: str, field: str, rendered: str) -> str:
|
||||
"""Surgically set `field: rendered` inside the YAML frontmatter block, creating the
|
||||
key (or the whole block) if absent. Every other line is preserved verbatim — this is
|
||||
the safety property that makes it OK to fall back from a failed PATCH."""
|
||||
line = f"{field}: {rendered}".rstrip()
|
||||
lines = text.splitlines()
|
||||
if lines and lines[0].strip() == "---":
|
||||
for end in range(1, len(lines)):
|
||||
if lines[end].strip() == "---":
|
||||
for i in range(1, end): # key exists -> replace that one line
|
||||
if re.match(rf"^{re.escape(field)}\s*:", lines[i]):
|
||||
lines[i] = line
|
||||
break
|
||||
else:
|
||||
lines.insert(end, line) # else append just before the closing ---
|
||||
return "\n".join(lines) + ("\n" if text.endswith("\n") else "")
|
||||
return f"---\n{line}\n---\n{text}" # no frontmatter at all -> prepend a minimal block
|
||||
|
||||
|
||||
def cmd_fm(path: str, field: str, value: str) -> int:
|
||||
"""Create-or-replace a frontmatter scalar. PATCH replace handles the common case;
|
||||
a missing key returns 400 invalid-target (PATCH cannot create), which used to make
|
||||
`fm` fail on exactly the notes that needed repair — now it falls back to a surgical
|
||||
GET -> insert-one-line -> verified PUT."""
|
||||
data = normalize_json_scalar(value)
|
||||
try:
|
||||
return cmd_patch(path, "replace", "frontmatter", field, temp_file(data))
|
||||
except ChorusError as exc:
|
||||
if "HTTP 400" not in str(exc):
|
||||
raise
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"fm {path}")
|
||||
new = _fm_upsert_line(body.decode(errors="replace"), field,
|
||||
_yaml_flow(json.loads(data.decode("utf-8"))))
|
||||
status, body = _qwrite("PUT", vault_url(path), data=new.encode("utf-8"),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
if status == 202:
|
||||
print(f"queued (offline): FM {field} -> {path} — will sync on the next reachable session")
|
||||
return 0
|
||||
check(status, body, f"fm {path}")
|
||||
if VERIFY:
|
||||
status, body = request("GET", vault_url(path))
|
||||
if status != 200 or not re.search(rf"(?m)^{re.escape(field)}\s*:", body.decode(errors="replace")):
|
||||
raise ChorusError(f"fm {path}: created key '{field}' did not verify on read-back")
|
||||
print(f"ok: FM created '{field}' in {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_delete(path: str) -> int:
|
||||
status, body = _qwrite("DELETE", vault_url(path))
|
||||
if status == 202:
|
||||
print(f"queued (offline): DELETE {path} — will sync on the next reachable session")
|
||||
return 0
|
||||
check(status, body, f"delete {path}")
|
||||
print(f"ok: DELETE {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_lock_time(value: str) -> int:
|
||||
try:
|
||||
return int(dt.datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
|
||||
.replace(tzinfo=dt.timezone.utc).timestamp())
|
||||
except Exception:
|
||||
# Unparseable timestamp -> treat the lock as FRESH (return "now"), so a
|
||||
# garbled lock is respected rather than silently stomped.
|
||||
return int(time.time())
|
||||
|
||||
|
||||
def cmd_lock(owner: str, quiet: bool = False) -> int:
|
||||
status, body = request("GET", vault_url(LOCK_PATH))
|
||||
if status == 200 and body.strip():
|
||||
current = body.decode(errors="replace").strip()
|
||||
held_owner, _, held_iso = current.partition(" @ ")
|
||||
if held_owner != owner and int(time.time()) - parse_lock_time(held_iso) < LOCK_TTL:
|
||||
print(f"lock held by '{held_owner}' since {held_iso} (fresh)", file=sys.stderr)
|
||||
return 75
|
||||
status, body = request("PUT", vault_url(LOCK_PATH), data=f"{owner} @ {now_iso()}\n".encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
check(status, body, "lock")
|
||||
# Read-back confirm we actually hold it. The REST API has no compare-and-swap,
|
||||
# so two racers can both PUT; last writer wins. Whoever does not own the file
|
||||
# on read-back backs off instead of proceeding under a false lock.
|
||||
status, body = request("GET", vault_url(LOCK_PATH))
|
||||
if status == 200:
|
||||
held_owner = body.decode(errors="replace").split(" @ ", 1)[0].strip()
|
||||
if held_owner != owner:
|
||||
print(f"lock race: now held by '{held_owner}', not '{owner}' — backing off", file=sys.stderr)
|
||||
return 75
|
||||
if not quiet:
|
||||
print(f"ok: locked by {owner}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_unlock(owner: str, quiet: bool = False) -> int:
|
||||
status, body = request("GET", vault_url(LOCK_PATH))
|
||||
if status == 200:
|
||||
held_owner = body.decode(errors="replace").split(" @ ", 1)[0].strip()
|
||||
if held_owner != owner:
|
||||
print(f"lock owned by '{held_owner}', not '{owner}' — not releasing", file=sys.stderr)
|
||||
return 75
|
||||
status, body = request("DELETE", vault_url(LOCK_PATH))
|
||||
if status != 404:
|
||||
check(status, body, "unlock")
|
||||
if not quiet:
|
||||
print("ok: unlocked")
|
||||
return 0
|
||||
|
||||
|
||||
def extract_heading(markdown: str, heading: str) -> str:
|
||||
"""Return the body under a `## <heading>` up to the next `## ` heading."""
|
||||
out: list[str] = []
|
||||
capture = False
|
||||
marker = f"## {heading}"
|
||||
for line in markdown.splitlines():
|
||||
if line.strip() == marker:
|
||||
capture = True
|
||||
continue
|
||||
if capture and line.startswith("## "):
|
||||
break
|
||||
if capture:
|
||||
out.append(line)
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
def cmd_scope(subcommand: str, text: str | None = None, as_json: bool = False) -> int:
|
||||
path = "_agent/context/current-context.md"
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"scope {subcommand}")
|
||||
current = body.decode(errors="replace")
|
||||
|
||||
if subcommand == "show":
|
||||
scope_text = extract_heading(current, "Scope")
|
||||
scope_updated = ""
|
||||
for line in current.splitlines():
|
||||
if line.startswith("scope_updated:"):
|
||||
scope_updated = line.split(":", 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
sessions_since = None
|
||||
# Count session logs dated after scope_updated (the drift signal).
|
||||
if scope_updated:
|
||||
status, body = request("GET", vault_url("_agent/sessions/"))
|
||||
if status == 200:
|
||||
try:
|
||||
files = json.loads(body).get("files", [])
|
||||
sessions_since = sum(1 for f in files if f.endswith(".md") and f[:10] > scope_updated)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if as_json:
|
||||
print(json.dumps({"ok": True, "action": "scope-show", "scope": scope_text,
|
||||
"scope_updated": scope_updated or None,
|
||||
"sessions_since": sessions_since}, ensure_ascii=False))
|
||||
return 0
|
||||
print("-- Active scope --")
|
||||
print(scope_text)
|
||||
print(f"scope_updated: {scope_updated or '<missing — drift cannot be detected; run scope set or repair>'}")
|
||||
if sessions_since is not None:
|
||||
print(f"sessions logged since: {sessions_since}")
|
||||
return 0
|
||||
|
||||
if subcommand != "set":
|
||||
raise ChorusError("scope: use 'show' or 'set \"<text>\"'", 2)
|
||||
if not text:
|
||||
raise ChorusError("scope set needs the new scope text", 2)
|
||||
prior = extract_heading(current, "Scope").replace("\n", " ").strip()[:140] or "(prior scope)"
|
||||
cmd_patch(path, "prepend", "heading", "Current Context::Scope History",
|
||||
temp_file(f"- {today()}: {prior}\n".encode()))
|
||||
cmd_patch(path, "replace", "heading", "Current Context::Scope",
|
||||
temp_file(f"{text}\n".encode()))
|
||||
try:
|
||||
cmd_patch(path, "replace", "frontmatter", "scope_updated",
|
||||
temp_file(json.dumps(today()).encode()))
|
||||
except ChorusError as exc:
|
||||
raise ChorusError(
|
||||
"scope set: body switched, but scope_updated frontmatter is missing "
|
||||
f"(run bootstrap.py to add it) [{exc}]"
|
||||
)
|
||||
print(f"ok: scope switched (prior archived to Scope History; scope_updated={today()})")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_load() -> int:
|
||||
"""Cold-start orientation: the canonical 6 reads in one call. 404s on today's
|
||||
daily note and the inbox are normal (printed as absent, not errors)."""
|
||||
if not chorus_config.is_configured(_CFG):
|
||||
print("chorus: NOT CONFIGURED — this machine has no usable CHORUS key file yet.")
|
||||
print(f"Expected at: {chorus_config.config_path()}")
|
||||
print("ACTION: ask the operator for their chorus-memory config file (it holds the")
|
||||
print(" vault owner, endpoint, and API key), then install it with ONE of:")
|
||||
print(" python3 chorus.py config import <path-to-their-file>")
|
||||
print(" python3 chorus.py config set --owner \"…\" --endpoint \"https://…\" --key \"…\"")
|
||||
print("Then re-run load. (Memory is unavailable until configured.)")
|
||||
return 78 # distinct: configuration required
|
||||
targets = [
|
||||
("marker", MARKER_PATH),
|
||||
("preferences", "_agent/memory/semantic/operator-preferences.md"),
|
||||
("context", "_agent/context/current-context.md"),
|
||||
("heartbeat", "_agent/heartbeat/last-session.md"),
|
||||
("today", f"journal/daily/{today()}.md"),
|
||||
("inbox", "inbox/captures/inbox.md"),
|
||||
]
|
||||
import chorus_queue
|
||||
# H2: sync any writes queued during a prior outage, best-effort and quiet on empty.
|
||||
try:
|
||||
synced = chorus_queue.flush()
|
||||
if synced:
|
||||
print(f"(synced {synced} write(s) queued during a prior offline session)\n")
|
||||
except Exception as exc: # noqa: BLE001 — never let queue upkeep block a load
|
||||
print(f"(queue flush skipped: {exc})\n", file=sys.stderr)
|
||||
|
||||
marker_missing = False
|
||||
heartbeat_absent = False
|
||||
offline = False
|
||||
for label, path in targets:
|
||||
if label == "marker":
|
||||
path, status, body = marker_get() # falls back to a legacy ECHO marker
|
||||
else:
|
||||
status, body = request("GET", vault_url(path))
|
||||
if status == 200:
|
||||
chorus_queue.cache_put(path, body) # refresh last-known-good for offline fallback
|
||||
print(f"===== {label}: {path} (HTTP 200) =====")
|
||||
text = body.decode(errors="replace")
|
||||
sys.stdout.write(text)
|
||||
if not text.endswith("\n"):
|
||||
sys.stdout.write("\n")
|
||||
elif status == 404:
|
||||
print(f"===== {label}: {path} (HTTP 404) =====")
|
||||
print("(absent — fine)")
|
||||
if label == "marker":
|
||||
marker_missing = True
|
||||
if label == "heartbeat":
|
||||
heartbeat_absent = True
|
||||
elif status == 0: # vault unreachable -> degrade to last-known-good cache
|
||||
offline = True
|
||||
cached = chorus_queue.cache_get(path)
|
||||
if cached is not None:
|
||||
print(f"===== {label}: {path} (OFFLINE — serving stale cache) =====")
|
||||
sys.stdout.write(cached.decode(errors="replace"))
|
||||
if not cached.endswith(b"\n"):
|
||||
sys.stdout.write("\n")
|
||||
else:
|
||||
print(f"===== {label}: {path} (OFFLINE — no cache) =====")
|
||||
print("(unreachable and not cached)")
|
||||
else:
|
||||
print(f"===== {label}: {path} (HTTP {status}) =====")
|
||||
print(f"(error HTTP {status}: {body.decode(errors='replace')[:200]})")
|
||||
print()
|
||||
# M3: heartbeat pointer missing/stale -> fall back to the recent sessions listing
|
||||
# (matches the documented loading procedure) so orientation works without the pointer.
|
||||
if heartbeat_absent and not offline:
|
||||
st, body = request("GET", vault_url("_agent/sessions/"))
|
||||
if st == 200:
|
||||
try:
|
||||
files = sorted((f for f in json.loads(body).get("files", []) if f.endswith(".md")),
|
||||
reverse=True)
|
||||
except json.JSONDecodeError:
|
||||
files = []
|
||||
if files:
|
||||
print("===== recent sessions (heartbeat absent — fallback) =====")
|
||||
for f in files[:5]:
|
||||
print(f" _agent/sessions/{f}")
|
||||
print()
|
||||
if offline:
|
||||
print("NOTE: vault unreachable — context above is last-known-good cache; writes this "
|
||||
"session will be queued and synced when the vault returns.")
|
||||
if marker_missing:
|
||||
print("NOTE: marker absent — vault not bootstrapped. Run bootstrap.py before relying on memory.")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_config(args) -> int:
|
||||
"""show | init | set the machine-local owner/endpoint/key config."""
|
||||
path = chorus_config.config_path()
|
||||
if args.action == "init":
|
||||
p, created = chorus_config.init_template()
|
||||
print(f"ok: wrote config template to {p} — edit it with your owner/endpoint/key"
|
||||
if created else f"config already exists at {p} (left unchanged)")
|
||||
return 0
|
||||
if args.action == "import":
|
||||
if not args.path:
|
||||
print("config import: pass the path to a config file "
|
||||
"(JSON with owner/endpoint/key)", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
p = chorus_config.import_file(args.path)
|
||||
except RuntimeError as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
return 2
|
||||
print(f"ok: imported config to {p} (chmod 600 best-effort)")
|
||||
return 0
|
||||
if args.action == "set":
|
||||
if args.owner is None and args.endpoint is None and args.key is None:
|
||||
print("config set: pass at least one of --owner / --endpoint / --key", file=sys.stderr)
|
||||
return 2
|
||||
p = chorus_config.write_config(args.owner, args.endpoint, args.key)
|
||||
print(f"ok: updated {p} (chmod 600 best-effort)")
|
||||
return 0
|
||||
# show — never prints the key in the clear
|
||||
cfg = chorus_config.load()
|
||||
key = cfg["key"]
|
||||
redacted = (key[:4] + "…" + key[-4:]) if len(key) >= 12 else ("set" if key else "")
|
||||
print(f"config file: {path}" + ("" if path.exists() else " (absent)"))
|
||||
for field, shown in (("owner", cfg["owner"]), ("endpoint", cfg["endpoint"]), ("key", redacted)):
|
||||
src = chorus_config.source(field)
|
||||
print(f" {field:9} {shown or '(unset)'} [{src}]")
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Validated cross-platform CHORUS vault client")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("load")
|
||||
sub.add_parser("flush") # H2: replay writes queued during a prior outage
|
||||
sub.add_parser("doctor") # M3: one-call readiness check
|
||||
sub.add_parser("get").add_argument("path")
|
||||
sub.add_parser("map").add_argument("path")
|
||||
sub.add_parser("ls").add_argument("path")
|
||||
sub.add_parser("search").add_argument("query", nargs="+")
|
||||
p = sub.add_parser("put"); p.add_argument("path"); p.add_argument("file", nargs="?")
|
||||
p = sub.add_parser("post"); p.add_argument("path"); p.add_argument("file", nargs="?")
|
||||
p = sub.add_parser("append"); p.add_argument("path"); p.add_argument("line")
|
||||
p = sub.add_parser("patch")
|
||||
p.add_argument("path"); p.add_argument("op"); p.add_argument("target_type")
|
||||
p.add_argument("target"); p.add_argument("file", nargs="?")
|
||||
p = sub.add_parser("fm"); p.add_argument("path"); p.add_argument("field"); p.add_argument("value")
|
||||
p = sub.add_parser("bump"); p.add_argument("path"); p.add_argument("date", nargs="?")
|
||||
sub.add_parser("delete").add_argument("path")
|
||||
sub.add_parser("lock").add_argument("owner")
|
||||
sub.add_parser("unlock").add_argument("owner")
|
||||
p = sub.add_parser("config") # machine-local owner/endpoint/key (see chorus_config)
|
||||
p.add_argument("action", nargs="?", default="show", choices=["show", "init", "set", "import"])
|
||||
p.add_argument("path", nargs="?") # for `config import <path>`
|
||||
p.add_argument("--owner"); p.add_argument("--endpoint"); p.add_argument("--key")
|
||||
p = sub.add_parser("scope")
|
||||
p.add_argument("subcommand", nargs="?", default="show", choices=["show", "set"])
|
||||
p.add_argument("text", nargs="?")
|
||||
p.add_argument("--json", action="store_true") # structured scope + freshness
|
||||
p = sub.add_parser("reflect") # H5: apply a JSON proposal set (dry-run unless --apply)
|
||||
p.add_argument("file", nargs="?")
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p = sub.add_parser("triage") # one-tap inbox triage (reflect pipeline + audit log)
|
||||
p.add_argument("file", nargs="?") # proposals JSON; omit to list
|
||||
p.add_argument("--list", action="store_true", dest="list_inbox")
|
||||
p.add_argument("--json", action="store_true") # structured inbox listing
|
||||
p.add_argument("--apply", action="store_true") # route + write processing log
|
||||
|
||||
# High-level operations (entity index + cross-links + recall). See chorus_ops.py.
|
||||
sub.add_parser("resolve").add_argument("mention", nargs="+")
|
||||
p = sub.add_parser("recall"); p.add_argument("query", nargs="+"); p.add_argument("--limit", type=int, default=6)
|
||||
p.add_argument("--json", action="store_true") # structured hits + neighbourhood
|
||||
p = sub.add_parser("link"); p.add_argument("a"); p.add_argument("b")
|
||||
p.add_argument("--json", action="store_true")
|
||||
p = sub.add_parser("capture")
|
||||
p.add_argument("title")
|
||||
p.add_argument("file", nargs="?")
|
||||
p.add_argument("--kind")
|
||||
p.add_argument("--inbox", action="store_true")
|
||||
p.add_argument("--status", default="")
|
||||
p.add_argument("--aliases", default="")
|
||||
p.add_argument("--source", default="")
|
||||
p.add_argument("--tags", default="") # extra tags; the kind is always seeded
|
||||
p.add_argument("--date")
|
||||
p.add_argument("--domain", default="business")
|
||||
p.add_argument("--no-log", action="store_true")
|
||||
p.add_argument("--json", action="store_true") # M4: emit a JSON result envelope
|
||||
p.add_argument("--dry-run", action="store_true") # M4: preview the plan, write nothing
|
||||
p.add_argument("--force", action="store_true") # create despite a duplicate-gate hit
|
||||
p.add_argument("--merge-into", metavar="SLUG") # route as an update to this entity
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.cmd == "load":
|
||||
return cmd_load()
|
||||
if args.cmd == "doctor":
|
||||
import chorus_doctor
|
||||
return chorus_doctor.run()
|
||||
if args.cmd == "flush":
|
||||
import chorus_queue
|
||||
n = chorus_queue.flush()
|
||||
remaining = len(chorus_queue.pending())
|
||||
if n:
|
||||
print(f"ok: flushed {n} queued write(s)"
|
||||
+ (f"; {remaining} still queued (vault unreachable)" if remaining else ""))
|
||||
elif remaining:
|
||||
print(f"ok: {remaining} write(s) still queued (vault unreachable)")
|
||||
else:
|
||||
print("ok: queue empty")
|
||||
return 0
|
||||
if args.cmd == "config":
|
||||
return cmd_config(args)
|
||||
if args.cmd == "get":
|
||||
return cmd_get(args.path)
|
||||
if args.cmd == "map":
|
||||
return cmd_map(args.path)
|
||||
if args.cmd == "ls":
|
||||
return cmd_ls(args.path)
|
||||
if args.cmd == "search":
|
||||
return cmd_search(args.query)
|
||||
if args.cmd == "put":
|
||||
return cmd_put(args.path, args.file)
|
||||
if args.cmd == "post":
|
||||
return cmd_post(args.path, args.file)
|
||||
if args.cmd == "append":
|
||||
return cmd_append(args.path, args.line)
|
||||
if args.cmd == "patch":
|
||||
return cmd_patch(args.path, args.op, args.target_type, args.target, args.file)
|
||||
if args.cmd == "fm":
|
||||
return cmd_fm(args.path, args.field, args.value)
|
||||
if args.cmd == "bump":
|
||||
return cmd_fm(args.path, "updated", json.dumps(args.date or today()))
|
||||
if args.cmd == "delete":
|
||||
return cmd_delete(args.path)
|
||||
if args.cmd == "lock":
|
||||
return cmd_lock(args.owner)
|
||||
if args.cmd == "unlock":
|
||||
return cmd_unlock(args.owner)
|
||||
if args.cmd == "scope":
|
||||
return cmd_scope(args.subcommand, args.text, as_json=args.json)
|
||||
if args.cmd == "reflect":
|
||||
import chorus_reflect
|
||||
raw = read_body(args.file).decode("utf-8", errors="replace")
|
||||
try:
|
||||
proposals = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ChorusError(f"reflect: proposals must be a JSON array ({exc})", 2)
|
||||
if not isinstance(proposals, list):
|
||||
raise ChorusError("reflect: proposals must be a JSON array of objects", 2)
|
||||
return chorus_reflect.apply(proposals, confirm=args.apply)
|
||||
if args.cmd == "triage":
|
||||
import chorus_triage
|
||||
if args.list_inbox or not args.file:
|
||||
return chorus_triage.list_inbox(as_json=args.json)
|
||||
raw = read_body(args.file).decode("utf-8", errors="replace")
|
||||
try:
|
||||
proposals = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ChorusError(f"triage: proposals must be a JSON array ({exc})", 2)
|
||||
if not isinstance(proposals, list):
|
||||
raise ChorusError("triage: proposals must be a JSON array of objects", 2)
|
||||
return chorus_triage.apply(proposals, confirm=args.apply)
|
||||
if args.cmd in ("resolve", "recall", "link", "capture"):
|
||||
import chorus_ops # lazy: only the high-level ops pull in the index/link modules
|
||||
if args.cmd == "resolve":
|
||||
return chorus_ops.resolve(" ".join(args.mention))
|
||||
if args.cmd == "recall":
|
||||
return chorus_ops.recall(args.query, limit=args.limit, as_json=args.json)
|
||||
if args.cmd == "link":
|
||||
return chorus_ops.link(args.a, args.b, as_json=args.json)
|
||||
return chorus_ops.capture(
|
||||
args.kind, args.title, args.file, status_v=args.status,
|
||||
aliases=args.aliases.split(",") if args.aliases else [],
|
||||
sources=args.source.split(",") if args.source else [],
|
||||
tags=args.tags.split(",") if args.tags else [],
|
||||
date=args.date, domain=args.domain, inbox=args.inbox, no_log=args.no_log,
|
||||
as_json=args.json, dry_run=args.dry_run,
|
||||
force=args.force, merge_into=args.merge_into)
|
||||
except ChorusError as exc:
|
||||
print(f"chorus.py: {exc}", file=sys.stderr)
|
||||
return exc.code
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_concurrency.py — H3: make the multi-writer story actually safe. [v1.0 SCAFFOLD — not yet wired]
|
||||
|
||||
The plugin advertises a shared vault (Claude Code + CoWork), but two real races exist:
|
||||
1. The entity index is a full-file load -> mutate -> PUT (chorus_index.load/save) with
|
||||
NO lock. Two concurrent `capture` calls both read the old index; last PUT wins;
|
||||
the other entity is silently dropped from the index (the note file survives, but
|
||||
resolve/recall miss it until the next sweep).
|
||||
2. The advisory lock is MANUAL (chorus.py cmd_lock) — capture / scope set never take it;
|
||||
it relies on the model remembering to.
|
||||
|
||||
This module provides (a) a context manager that auto-acquires/releases the existing
|
||||
advisory lock around a write burst, with crash-safe release, and (b) an atomic
|
||||
index-update that re-reads UNDER the lock before saving, so concurrent captures merge
|
||||
instead of clobber.
|
||||
|
||||
ACCEPTANCE (v1.0 roadmap H3, shipped; roadmap retired — see git history):
|
||||
- two concurrent captures both land in the index (no lost entries)
|
||||
- lock auto-released on normal AND error exit
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
import chorus_index as idx_mod # noqa: E402
|
||||
|
||||
|
||||
def auto_owner() -> str:
|
||||
"""A stable-per-process owner id: client tag + pid. Avoids Math.random/time-based
|
||||
ids; pid is unique enough for cooperative locking and is reproducible in tests."""
|
||||
tag = os.environ.get("CHORUS_CLIENT", "cc")
|
||||
return f"{tag}-{os.getpid()}"
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def vault_lock(owner: str | None = None, required: bool = False):
|
||||
"""Hold the advisory lock for a write burst; release on exit (incl. exceptions).
|
||||
|
||||
Yields `held: bool`. The lock is cooperative (read-back-confirmed, TTL-reclaimable),
|
||||
so if another session holds it we do NOT block: with `required=False` we yield
|
||||
`held=False` and let the caller proceed (advisory) or warn; with `required=True` we
|
||||
raise. Release is best-effort and never masks the body's own exception.
|
||||
"""
|
||||
owner = owner or auto_owner()
|
||||
rc = chorus.cmd_lock(owner, quiet=True)
|
||||
held = rc == 0
|
||||
if not held and required:
|
||||
raise chorus.ChorusError(f"vault_lock: could not acquire lock for '{owner}' (held elsewhere)", 75)
|
||||
try:
|
||||
yield held
|
||||
finally:
|
||||
if held:
|
||||
try:
|
||||
chorus.cmd_unlock(owner, quiet=True)
|
||||
except Exception: # noqa: BLE001 — release is best-effort; TTL reclaims a leak
|
||||
pass
|
||||
|
||||
|
||||
def atomic_index_update(mutate, owner: str | None = None):
|
||||
"""Apply `mutate(index)` to the entity index without losing a concurrent writer's
|
||||
entry: take the lock, **re-read the index fresh** (not a stale in-memory copy),
|
||||
run the mutator, save, release. Returns the freshly-read-and-saved index.
|
||||
|
||||
The fresh re-read inside the critical section is the core of the fix — even if the
|
||||
advisory lock can't be taken (another session active), reading immediately before
|
||||
the save collapses the read-modify-write window to two calls instead of spanning the
|
||||
whole capture, so an unlocked update is still far safer than the prior code.
|
||||
"""
|
||||
with vault_lock(owner) as held:
|
||||
index = idx_mod.load()
|
||||
mutate(index)
|
||||
idx_mod.save(index)
|
||||
if not held:
|
||||
print("chorus_concurrency: entity index updated WITHOUT the advisory lock "
|
||||
"(another session may be active); the fresh re-read minimizes the race.",
|
||||
file=sys.stderr)
|
||||
return index
|
||||
@@ -0,0 +1,266 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_config.py — resolve the vault owner, endpoint, and API key for CHORUS.
|
||||
|
||||
The plugin ships NO owner, endpoint, or key — it is fully user-agnostic. On each
|
||||
machine, a machine-local JSON config supplies them:
|
||||
|
||||
~/.claude/chorus-memory/config.json
|
||||
{
|
||||
"owner": "Full Name (how the agent refers to the vault owner, third person)",
|
||||
"endpoint": "https://your-obsidian-rest-endpoint.example.com",
|
||||
"key": "<obsidian-local-rest-api-bearer-token>"
|
||||
}
|
||||
|
||||
This file is per-machine: it must be created on each fresh install, is never
|
||||
committed, and is never written into a vault note. To create it, run
|
||||
`python3 chorus.py config init` (writes a placeholder template when absent) and edit
|
||||
it, or `python3 chorus.py config set --owner ... --endpoint ... --key ...`.
|
||||
|
||||
Resolution — a COMPLETE baked credential set wins unconditionally:
|
||||
|
||||
If the artifact carries both a baked endpoint AND key (a per-user build), those
|
||||
baked values are used as-is and CANNOT be shadowed by env vars or a config file.
|
||||
This is the default path: a delivered per-user plugin "just works" with no setup,
|
||||
and a stale CHORUS_* env var or a leftover/placeholder ~/.claude config.json can
|
||||
never override or break it.
|
||||
|
||||
Otherwise (nothing baked — the generic published plugin), fall back per-field,
|
||||
first hit wins:
|
||||
owner env CHORUS_OWNER -> config "owner"
|
||||
endpoint env CHORUS_BASE -> config "endpoint"
|
||||
key env CHORUS_KEY -> config "key"
|
||||
|
||||
The config directory honors $CLAUDE_CONFIG_DIR (defaults to ~/.claude); the config
|
||||
file path itself can be overridden with $CHORUS_CONFIG. Pure stdlib only.
|
||||
|
||||
BAKED CREDENTIALS (the default tier): the DEFAULT_* constants below are EMPTY in
|
||||
source — the committed plugin ships zero credentials. `build.py --bake-key`
|
||||
substitutes a specific user's owner/endpoint/key into them at package time,
|
||||
producing a per-user artifact that needs no config file. This is how the plugin
|
||||
works in the CoWork sandbox, where the user's ~/.claude is not bridged: the plugin
|
||||
itself (mounted read-only every session) carries the values. A baked artifact is
|
||||
secret-bearing — deliver it directly to that one user, NEVER commit or publish it.
|
||||
On a normal host the empty defaults mean the env/config-file path takes over.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Build-time injection targets — MUST stay empty string literals in source so the
|
||||
# committed tree ships no secret. `build.py --bake-key` rewrites these lines.
|
||||
DEFAULT_OWNER = ""
|
||||
DEFAULT_BASE = ""
|
||||
DEFAULT_KEY = ""
|
||||
|
||||
# ---------------------------------------------------------------- legacy shim
|
||||
# CHORUS is a fork of ECHO. For one major version, legacy ECHO_* environment
|
||||
# variables keep working: at import time each ECHO_<X> is adopted as CHORUS_<X>
|
||||
# whenever CHORUS_<X> is not itself set. Scripts only ever read CHORUS_*.
|
||||
_LEGACY_ENV_SUFFIXES = (
|
||||
"OWNER", "BASE", "KEY", "CONFIG", "TODAY", "VERIFY", "LOCK_TTL", "TIMEOUT",
|
||||
"WORKERS", "STATE_DIR", "CLIENT", "DUP_GATE", "REFLECT_MIN_TURNS",
|
||||
"FRESH_HALF_LIFE",
|
||||
)
|
||||
|
||||
|
||||
def _alias_legacy_env() -> None:
|
||||
for suffix in _LEGACY_ENV_SUFFIXES:
|
||||
new, old = f"CHORUS_{suffix}", f"ECHO_{suffix}"
|
||||
if not os.environ.get(new) and os.environ.get(old):
|
||||
os.environ[new] = os.environ[old]
|
||||
|
||||
|
||||
_alias_legacy_env()
|
||||
|
||||
TEMPLATE = {
|
||||
"owner": "Full Name — how the agent should refer to the vault owner (third person)",
|
||||
"endpoint": "https://your-obsidian-rest-endpoint.example.com",
|
||||
"key": "<obsidian-local-rest-api-bearer-token>",
|
||||
}
|
||||
|
||||
|
||||
def claude_dir() -> Path:
|
||||
"""The Claude config directory ($CLAUDE_CONFIG_DIR, else ~/.claude)."""
|
||||
return Path(os.environ.get("CLAUDE_CONFIG_DIR") or (Path.home() / ".claude"))
|
||||
|
||||
|
||||
def config_path() -> Path:
|
||||
"""Absolute path to the machine-local config file (the CHORUS path — this is
|
||||
where writes always land)."""
|
||||
override = os.environ.get("CHORUS_CONFIG")
|
||||
if override:
|
||||
return Path(override).expanduser()
|
||||
return claude_dir() / "chorus-memory" / "config.json"
|
||||
|
||||
|
||||
def legacy_config_path() -> Path:
|
||||
"""The pre-fork ECHO config location, honored read-only as a fallback."""
|
||||
return claude_dir() / "echo-memory" / "config.json"
|
||||
|
||||
|
||||
def read_config_path() -> Path:
|
||||
"""The config file to READ: the CHORUS path when overridden or present,
|
||||
else a legacy ECHO config if one exists (fork adoption path). Writes never
|
||||
go to the legacy path — `config set`/`import` land at config_path()."""
|
||||
p = config_path()
|
||||
if os.environ.get("CHORUS_CONFIG") or p.exists():
|
||||
return p
|
||||
legacy = legacy_config_path()
|
||||
return legacy if legacy.exists() else p
|
||||
|
||||
|
||||
def _from_file() -> dict:
|
||||
p = read_config_path()
|
||||
if not p.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def baked_complete() -> bool:
|
||||
"""True when the artifact carries a usable baked endpoint AND key — i.e. a
|
||||
per-user `--bake-key` build. When True, the baked values win unconditionally."""
|
||||
return bool(DEFAULT_BASE and DEFAULT_KEY)
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
"""Return {owner, endpoint, key}. A complete baked set wins unconditionally;
|
||||
otherwise env overrides the file, missing -> "".
|
||||
|
||||
Never raises — callers that REQUIRE a field use resolve_endpoint()/resolve_key(),
|
||||
which raise a helpful error. This keeps `--help`, `config`, and `doctor` usable
|
||||
even when nothing is configured yet."""
|
||||
f = _from_file()
|
||||
if baked_complete():
|
||||
# Per-user artifact: baked creds are authoritative and must not be shadowed
|
||||
# by a stale env var or a leftover/placeholder config file. Owner is purely
|
||||
# descriptive, so it may still fall back when not baked.
|
||||
return {
|
||||
"owner": DEFAULT_OWNER or os.environ.get("CHORUS_OWNER") or f.get("owner") or "",
|
||||
"endpoint": DEFAULT_BASE.rstrip("/"),
|
||||
"key": DEFAULT_KEY,
|
||||
}
|
||||
endpoint = (os.environ.get("CHORUS_BASE") or f.get("endpoint") or "").rstrip("/")
|
||||
return {
|
||||
"owner": os.environ.get("CHORUS_OWNER") or f.get("owner") or "",
|
||||
"endpoint": endpoint,
|
||||
"key": os.environ.get("CHORUS_KEY") or f.get("key") or "",
|
||||
}
|
||||
|
||||
|
||||
def _missing(field: str, env: str) -> RuntimeError:
|
||||
return RuntimeError(
|
||||
f"chorus: no {field} configured. Set {env}, or create {config_path()} "
|
||||
f"(run `chorus.py config init` to scaffold it, then edit). "
|
||||
f'Expected JSON keys: "owner", "endpoint", "key".'
|
||||
)
|
||||
|
||||
|
||||
def resolve_endpoint() -> str:
|
||||
ep = load()["endpoint"]
|
||||
if not ep:
|
||||
raise _missing("endpoint", "CHORUS_BASE")
|
||||
return ep
|
||||
|
||||
|
||||
def resolve_key() -> str:
|
||||
k = load()["key"]
|
||||
if not k:
|
||||
raise _missing("key", "CHORUS_KEY")
|
||||
return k
|
||||
|
||||
|
||||
def resolve_owner() -> str:
|
||||
"""The owner display name. Descriptive only, so an empty value is tolerated."""
|
||||
return load()["owner"]
|
||||
|
||||
|
||||
def is_configured(cfg: dict | None = None) -> bool:
|
||||
"""True only when endpoint AND key are present and NOT the untouched template
|
||||
placeholders — so a freshly `config init`-ed (but unedited) file reads as not yet
|
||||
configured, which is what triggers the first-run "ask for the key file" prompt."""
|
||||
c = cfg if cfg is not None else load()
|
||||
ep, key = c.get("endpoint", ""), c.get("key", "")
|
||||
if not ep or not key:
|
||||
return False
|
||||
if "your-obsidian-rest-endpoint" in ep or key.startswith("<"):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def source(field: str) -> str:
|
||||
"""Where a field is currently coming from — for `config`/`doctor` reporting."""
|
||||
baked = {"owner": DEFAULT_OWNER, "endpoint": DEFAULT_BASE, "key": DEFAULT_KEY}[field]
|
||||
# A complete baked set wins unconditionally (see load()): endpoint/key always
|
||||
# come from the artifact, and owner too whenever it was baked.
|
||||
if baked_complete() and (field in ("endpoint", "key") or baked):
|
||||
return "baked-in (plugin)"
|
||||
env = {"owner": "CHORUS_OWNER", "endpoint": "CHORUS_BASE", "key": "CHORUS_KEY"}[field]
|
||||
if os.environ.get(env):
|
||||
return f"{env} env"
|
||||
rp = read_config_path()
|
||||
if rp.exists() and _from_file().get(field):
|
||||
return "config file" if rp == config_path() else "config file (legacy echo-memory path)"
|
||||
if baked:
|
||||
return "baked-in (plugin)"
|
||||
return "unset"
|
||||
|
||||
|
||||
def template_text() -> str:
|
||||
return json.dumps(TEMPLATE, indent=2, ensure_ascii=False) + "\n"
|
||||
|
||||
|
||||
def _secure(p: Path) -> None:
|
||||
try:
|
||||
os.chmod(p, 0o600) # advisory on filesystems without POSIX mode bits (e.g. Windows)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def write_config(owner: str, endpoint: str, key: str) -> Path:
|
||||
"""Persist a filled-in config (merging over any existing values), chmod 0600."""
|
||||
cur = _from_file()
|
||||
merged = {
|
||||
"owner": owner if owner is not None else cur.get("owner", ""),
|
||||
"endpoint": (endpoint if endpoint is not None else cur.get("endpoint", "")).rstrip("/"),
|
||||
"key": key if key is not None else cur.get("key", ""),
|
||||
}
|
||||
p = config_path()
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(json.dumps(merged, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
_secure(p)
|
||||
return p
|
||||
|
||||
|
||||
def import_file(src) -> Path:
|
||||
"""Adopt a config file someone was handed: validate it has a usable endpoint+key,
|
||||
then copy it into config_path(). The 'provide the file -> plugin installs it' path."""
|
||||
p = Path(src).expanduser()
|
||||
if not p.exists():
|
||||
raise RuntimeError(f"config import: file not found: {p}")
|
||||
try:
|
||||
data = json.loads(p.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
raise RuntimeError(f"config import: {p} is not valid JSON ({exc})")
|
||||
if not isinstance(data, dict) or not is_configured(data):
|
||||
raise RuntimeError(
|
||||
'config import: file must be JSON with a real "endpoint" and "key" '
|
||||
'(and ideally "owner") — not the template placeholders.')
|
||||
return write_config(data.get("owner", ""), data["endpoint"], data["key"])
|
||||
|
||||
|
||||
def init_template() -> tuple[Path, bool]:
|
||||
"""Write the placeholder template to config_path() if absent.
|
||||
Returns (path, created) — created is False when a config already exists."""
|
||||
p = config_path()
|
||||
if p.exists():
|
||||
return p, False
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_text(template_text(), encoding="utf-8")
|
||||
_secure(p)
|
||||
return p, True
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_doctor.py — M3: one-call readiness check. [v1.0 SCAFFOLD — run() implemented]
|
||||
|
||||
There is no single "is everything OK" command. doctor checks, in order, the things
|
||||
that make CHORUS usable this session and prints a green/red report. Intended to back an
|
||||
`chorus.py doctor` subcommand and the start of /chorus-load when something looks wrong.
|
||||
|
||||
Also tracked here (TODO): complete the `load` fallback — chorus.cmd_load reads the
|
||||
heartbeat pointer but never lists _agent/sessions/ when it's missing/stale, though
|
||||
SKILL.md:122 documents that fallback. See patch_load_fallback() below.
|
||||
|
||||
Exit: 0 all green · 1 a check is red.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
|
||||
MIN_PY = (3, 9)
|
||||
|
||||
|
||||
def run() -> int:
|
||||
reds = 0
|
||||
|
||||
def line(ok: bool, label: str, detail: str = "") -> None:
|
||||
nonlocal reds
|
||||
reds += 0 if ok else 1
|
||||
print(f" [{'OK ' if ok else 'RED'}] {label}" + (f" — {detail}" if detail else ""))
|
||||
|
||||
import chorus_config
|
||||
cfg = chorus_config.load()
|
||||
print(f"chorus doctor — endpoint {cfg['endpoint'] or '(not configured)'}")
|
||||
|
||||
# 1. interpreter
|
||||
line(sys.version_info >= MIN_PY, f"python >= {MIN_PY[0]}.{MIN_PY[1]}",
|
||||
f"running {sys.version_info.major}.{sys.version_info.minor}")
|
||||
|
||||
# 2. machine-local config (owner/endpoint/key). Without a usable endpoint+key
|
||||
# nothing else can run, so report it clearly before touching the network.
|
||||
# A still-placeholder `config init` template counts as not configured.
|
||||
ep_real = bool(cfg["endpoint"]) and "your-obsidian-rest-endpoint" not in cfg["endpoint"]
|
||||
key_real = bool(cfg["key"]) and not cfg["key"].startswith("<")
|
||||
line(ep_real, "endpoint configured",
|
||||
f"[{chorus_config.source('endpoint')}]" if ep_real
|
||||
else (f"placeholder — edit {chorus_config.config_path()}" if cfg["endpoint"]
|
||||
else f"create {chorus_config.config_path()} (run `chorus.py config init`)"))
|
||||
line(key_real, "API key configured",
|
||||
f"[{chorus_config.source('key')}]" if key_real
|
||||
else ("placeholder — not yet filled in" if cfg["key"] else "missing — see `chorus.py config`"))
|
||||
line(bool(cfg["owner"]), "vault owner set",
|
||||
cfg["owner"] if cfg["owner"] else "optional, but recommended for third-person writes")
|
||||
if not chorus_config.is_configured(cfg):
|
||||
print("\ndoctor: not configured — ask the operator for their key file and install it "
|
||||
"(`chorus.py config import <file>`, or `config set --owner … --endpoint … --key …`), "
|
||||
"then re-run.")
|
||||
return 1
|
||||
|
||||
# 3. reachability + auth + marker, in one GET of the bootstrap marker
|
||||
# (marker_get falls back to a pre-fork ECHO marker on adopted vaults)
|
||||
marker_path, status, body = chorus.marker_get()
|
||||
if status == 0:
|
||||
line(False, "vault reachable", body.decode(errors="replace")[:120])
|
||||
print("\ndoctor: endpoint unreachable — is Obsidian + the Local REST API running?")
|
||||
return 1
|
||||
line(status not in (401, 403), "auth accepted", f"HTTP {status}" if status in (401, 403) else "")
|
||||
if status == 404:
|
||||
line(False, "vault bootstrapped", "marker absent — run bootstrap.py")
|
||||
elif status == 200:
|
||||
text = body.decode(errors="replace")
|
||||
ver = next((ln.split(":", 1)[1].strip() for ln in text.splitlines()
|
||||
if ln.startswith("schema_version:")), "?")
|
||||
legacy = " (legacy ECHO marker — run migrate.py to adopt)" if marker_path == chorus.LEGACY_MARKER_PATH else ""
|
||||
line(True, "vault bootstrapped", f"schema_version={ver}{legacy}")
|
||||
else:
|
||||
line(status < 400, "marker fetch", f"HTTP {status}")
|
||||
|
||||
# 4. invariants pointer.
|
||||
print(" [i] invariants — run `/chorus-health` (vault_lint.py) for the full check")
|
||||
|
||||
print(f"\ndoctor: {'all green' if reds == 0 else f'{reds} issue(s) — see RED above'}")
|
||||
return 1 if reds else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(run())
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_hook_session_start.py — SessionStart hook: self-firing cold-start load.
|
||||
|
||||
The two behaviours the skill cares most about (load at session start, reflect at
|
||||
session end) used to depend entirely on the model remembering to do them — scope
|
||||
drift and the inbox backlog are the residue of that discipline failing. This hook
|
||||
makes loading deterministic: on session start it runs the canonical `chorus.py load`
|
||||
and hands the output to the model as additionalContext.
|
||||
|
||||
Contract (a hook must NEVER break a session):
|
||||
* always exits 0 — any failure degrades to a one-line note or to silence;
|
||||
* fires only on `startup` / `clear` (resume & compact already carry context);
|
||||
* NOT CONFIGURED (exit 78) becomes a short instruction to run the first-run flow,
|
||||
not an error;
|
||||
* output is capped so a huge vault can't blow up the context window.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
MAX_CONTEXT = 16000 # chars of load output forwarded into the session context
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = json.load(sys.stdin)
|
||||
except Exception: # noqa: BLE001
|
||||
payload = {}
|
||||
if payload.get("source", "startup") not in ("startup", "clear"):
|
||||
return 0
|
||||
|
||||
header = ("[chorus-memory] Cold-start memory load (SessionStart hook). Reconcile per "
|
||||
"the chorus-memory skill: check inbox depth and confirm the scope still "
|
||||
"matches this session's request before working.\n\n")
|
||||
buf = io.StringIO()
|
||||
try:
|
||||
import chorus
|
||||
with contextlib.redirect_stdout(buf):
|
||||
rc = chorus.cmd_load()
|
||||
text = buf.getvalue()
|
||||
if rc == 78:
|
||||
context = ("[chorus-memory] CHORUS is NOT CONFIGURED on this machine. Before "
|
||||
"doing memory work, ask the operator for their chorus-memory config "
|
||||
"(owner/endpoint/key) and install it via `chorus.py config import "
|
||||
"<file>` — see the skill's First-run section.")
|
||||
elif rc == 0 and text.strip():
|
||||
if len(text) > MAX_CONTEXT:
|
||||
text = text[:MAX_CONTEXT] + "\n[... load output truncated by the hook ...]"
|
||||
context = header + text
|
||||
else:
|
||||
context = ("[chorus-memory] `chorus.py load` returned nothing usable "
|
||||
f"(exit {rc}) — run /chorus-doctor if memory seems unavailable.")
|
||||
except Exception as exc: # noqa: BLE001 — never break session start
|
||||
context = (f"[chorus-memory] SessionStart load skipped ({exc}). The vault may be "
|
||||
"unreachable; proceed without memory and mention it once if relevant.")
|
||||
|
||||
print(json.dumps({"hookSpecificOutput": {
|
||||
"hookEventName": "SessionStart",
|
||||
"additionalContext": context,
|
||||
}}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_hook_stop.py — Stop hook: one-shot session-end reflection nudge.
|
||||
|
||||
Reflection (/chorus-reflect) only captures memory if someone remembers to run it.
|
||||
This hook watches the first Stop of a *substantive* session and, when nothing has
|
||||
been reflected or session-logged yet, blocks once with a reminder so the model
|
||||
runs the reflect flow (which still previews and asks the operator before writing
|
||||
— the safety gate lives in reflect, not here).
|
||||
|
||||
Guards (a hook must NEVER trap a session in a loop or nag):
|
||||
* `stop_hook_active` -> exit immediately (loop guard);
|
||||
* fires at most ONCE per session (marker file under CHORUS_STATE_DIR/hook-state);
|
||||
* skips quiet sessions (< CHORUS_REFLECT_MIN_TURNS user turns, default 5);
|
||||
* skips when the transcript shows reflection / a session log already happened;
|
||||
* skips when CHORUS isn't configured on this machine;
|
||||
* always exits 0.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
MIN_TURNS = int(os.environ.get("CHORUS_REFLECT_MIN_TURNS", "5"))
|
||||
|
||||
REASON = (
|
||||
"[chorus-memory] Session-end reflection has not run yet. If this session produced "
|
||||
"durable facts, decisions, commitments, or artifacts worth remembering: run the "
|
||||
"/chorus-reflect flow (extract -> preview -> apply only with the operator's confirm) "
|
||||
"and write the session log + heartbeat per the chorus-memory skill. If nothing "
|
||||
"durable emerged, simply finish — do not invent memories. "
|
||||
"(This reminder fires once per session.)"
|
||||
)
|
||||
|
||||
|
||||
def state_marker(session_id: str) -> Path:
|
||||
# Mirrors chorus_queue.state_dir() (this hook runs standalone, so the
|
||||
# ECHO->CHORUS env aliasing in chorus_config may not have run): env first
|
||||
# (either name), else ~/.chorus-memory, honoring a pre-fork ~/.echo-memory
|
||||
# when the new dir doesn't exist yet.
|
||||
override = os.environ.get("CHORUS_STATE_DIR") or os.environ.get("ECHO_STATE_DIR")
|
||||
if override:
|
||||
base = Path(override)
|
||||
else:
|
||||
new = Path.home() / ".chorus-memory"
|
||||
legacy = Path.home() / ".echo-memory"
|
||||
base = legacy if (not new.exists() and legacy.exists()) else new
|
||||
return base / "hook-state" / f"{session_id}.reflect-nudged"
|
||||
|
||||
|
||||
def count_user_turns(raw: str) -> int:
|
||||
turns = 0
|
||||
for line in raw.splitlines():
|
||||
try:
|
||||
rec = json.loads(line)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if rec.get("type") != "user":
|
||||
continue
|
||||
content = (rec.get("message") or {}).get("content")
|
||||
if isinstance(content, str) and content.strip():
|
||||
turns += 1
|
||||
elif isinstance(content, list) and any(
|
||||
isinstance(b, dict) and b.get("type") == "text" for b in content):
|
||||
turns += 1
|
||||
return turns
|
||||
|
||||
|
||||
def already_reflected(raw: str) -> bool:
|
||||
"""Transcript evidence that reflection or session logging already happened."""
|
||||
return ("/chorus-reflect" in raw
|
||||
or re.search(r'chorus\.py"?\s+reflect\b', raw) is not None
|
||||
or "put _agent/sessions/" in raw
|
||||
or "put _agent/heartbeat/last-session.md" in raw)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
payload = json.load(sys.stdin)
|
||||
except Exception: # noqa: BLE001
|
||||
return 0
|
||||
if payload.get("stop_hook_active"):
|
||||
return 0
|
||||
session_id = str(payload.get("session_id") or "unknown")
|
||||
marker = state_marker(session_id)
|
||||
if marker.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
import chorus_config
|
||||
if not chorus_config.is_configured(chorus_config.load()):
|
||||
return 0 # nothing to reflect into — stay silent
|
||||
except Exception: # noqa: BLE001
|
||||
return 0
|
||||
|
||||
raw = ""
|
||||
tp = payload.get("transcript_path")
|
||||
if tp:
|
||||
try:
|
||||
raw = Path(tp).read_text(encoding="utf-8", errors="replace")
|
||||
except Exception: # noqa: BLE001
|
||||
raw = ""
|
||||
|
||||
if already_reflected(raw):
|
||||
try: # done for this session — never fire
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
marker.write_text("reflected\n", encoding="utf-8")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 0
|
||||
if count_user_turns(raw) < MIN_TURNS:
|
||||
return 0 # quiet so far; a later stop may qualify (no marker written)
|
||||
|
||||
try:
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
marker.write_text("nudged\n", encoding="utf-8")
|
||||
except Exception: # noqa: BLE001
|
||||
pass # if the marker can't be written, still nudge; stop_hook_active guards loops
|
||||
print(json.dumps({"decision": "block", "reason": REASON}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,323 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_index.py — the CHORUS entity index.
|
||||
|
||||
A small machine-readable registry at `_agent/index/entities.json` that turns
|
||||
routing/resolve into an O(1) lookup (no fuzzy search per write) and supplies the
|
||||
name->path map used for cross-linking and recall.
|
||||
|
||||
{
|
||||
"schema": 1,
|
||||
"updated": "YYYY-MM-DD",
|
||||
"entities": {
|
||||
"<slug>": {"path": "...", "kind": "...", "title": "...",
|
||||
"aliases": [...], "last_seen": "YYYY-MM-DD"}
|
||||
}
|
||||
}
|
||||
|
||||
Imported by chorus_ops.py, sweep.py, and vault_lint.py. Network goes through chorus.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
import chorus_quality # noqa: E402 — distinctive-token guards for fuzzy candidate matching
|
||||
|
||||
INDEX_PATH = "_agent/index/entities.json"
|
||||
|
||||
# kind -> destination folder (the canonical home for that entity kind)
|
||||
KIND_FOLDER = {
|
||||
"person": "resources/people",
|
||||
"company": "resources/companies",
|
||||
"concept": "resources/concepts",
|
||||
"reference": "resources/references",
|
||||
"meeting": "resources/meetings",
|
||||
"project": "projects/active", # new projects default to active
|
||||
"area": "areas", # needs a domain segment
|
||||
"semantic": "_agent/memory/semantic",
|
||||
"episodic": "_agent/memory/episodic",
|
||||
"working": "_agent/memory/working",
|
||||
"skill": "_agent/skills/active",
|
||||
"decision": "decisions/by-date",
|
||||
}
|
||||
# kind -> frontmatter `type:` value
|
||||
KIND_TYPE = {
|
||||
"person": "person", "company": "company", "concept": "concept",
|
||||
"reference": "reference", "meeting": "meeting", "project": "project",
|
||||
"area": "area", "semantic": "semantic-memory", "episodic": "episodic-memory",
|
||||
"working": "working-memory", "skill": "skill", "decision": "decision",
|
||||
}
|
||||
DATE_KINDS = {"meeting", "decision"} # filename carries a YYYY-MM-DD prefix
|
||||
AREA_DOMAINS = ("business", "personal", "learning", "systems")
|
||||
|
||||
# kind -> default `status:` stamped by capture when none is given. Every entity note is
|
||||
# born with a status — a missing key is the #1 source of frontmatter drift (18/71 notes
|
||||
# in the field audit). Records of past events default to done; living notes to active.
|
||||
KIND_STATUS = {
|
||||
"person": "active", "company": "active", "concept": "active",
|
||||
"reference": "active", "meeting": "done", "project": "active",
|
||||
"area": "active", "semantic": "active", "episodic": "done",
|
||||
"working": "active", "skill": "active", "decision": "done",
|
||||
}
|
||||
|
||||
# kind -> frontmatter fields that must be present AND non-empty on that kind's notes.
|
||||
# Data-driven so vault_lint's incomplete-frontmatter check and sweep's backfill stay in
|
||||
# lockstep with what capture stamps; add a (kind, field) pair here and all three follow.
|
||||
KIND_REQUIRED_FM = {
|
||||
"person": ("status", "tags"),
|
||||
"company": ("status", "tags"),
|
||||
"concept": ("status", "tags"),
|
||||
"reference": ("status", "tags"),
|
||||
"project": ("status", "tags"),
|
||||
"area": ("status", "tags"),
|
||||
"skill": ("status", "tags"),
|
||||
"meeting": ("tags",),
|
||||
"decision": ("tags",),
|
||||
}
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
text = str(text).strip().lower()
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
|
||||
return text[:40] or "untitled"
|
||||
|
||||
|
||||
def derive_aliases(title: str) -> list[str]:
|
||||
"""Safe, lossless name variants of a title — case/punctuation normalizations that are
|
||||
unambiguously the SAME name (hyphen<->space, lowercased). Deliberately NOT acronyms or
|
||||
token subsets: those collide across entities and would make resolve() match too much.
|
||||
Lets `chorus-memory` and `chorus memory` both resolve to a title written either way."""
|
||||
t = str(title or "").strip()
|
||||
if not t:
|
||||
return []
|
||||
low = t.lower()
|
||||
variants = {low, slugify(t), slugify(t).replace("-", " "),
|
||||
low.replace("-", " "), low.replace(" ", "-")}
|
||||
return sorted(v for v in variants if v)
|
||||
|
||||
|
||||
_WORD = re.compile(r"[a-z0-9]+")
|
||||
|
||||
|
||||
def _sig_tokens(s) -> set[str]:
|
||||
"""Distinctive tokens of a string: long enough and not a common word (reuses the
|
||||
auto-link guards), so 'data'/'api'/'the' can't drive a fuzzy match on their own."""
|
||||
return {t for t in _WORD.findall(str(s or "").lower())
|
||||
if len(t) >= chorus_quality.MIN_NAME_LEN and t not in chorus_quality._COMMON}
|
||||
|
||||
|
||||
def aliases_in_frontmatter(text: str) -> list[str]:
|
||||
"""Read an `aliases:` list (flow `[a, b]` or block `- a`) from a note's YAML frontmatter.
|
||||
Frontmatter is the durable, Obsidian-native home for aliases; sweep folds these into the
|
||||
index so they survive an index rebuild."""
|
||||
if not str(text).startswith("---"):
|
||||
return []
|
||||
end = text.find("\n---", 3)
|
||||
fm = text[:end] if end != -1 else text
|
||||
flow = re.search(r"(?m)^aliases:\s*\[(.*?)\]", fm)
|
||||
if flow:
|
||||
return [a.strip().strip('"').strip("'") for a in flow.group(1).split(",") if a.strip()]
|
||||
block = re.search(r"(?m)^aliases:\s*\n((?:[ \t]*-[ \t]*.+\n?)+)", fm)
|
||||
if block:
|
||||
return [ln.strip().lstrip("-").strip().strip('"').strip("'")
|
||||
for ln in block.group(1).splitlines() if ln.strip()]
|
||||
return []
|
||||
|
||||
|
||||
def derive_path(kind: str, slug: str, date: str | None = None, domain: str = "business") -> str:
|
||||
folder = KIND_FOLDER.get(kind)
|
||||
if folder is None:
|
||||
raise chorus.ChorusError(f"unknown kind '{kind}' (one of: {', '.join(sorted(KIND_FOLDER))})", 2)
|
||||
if kind == "area":
|
||||
if domain not in AREA_DOMAINS:
|
||||
raise chorus.ChorusError(f"area domain must be one of {AREA_DOMAINS}", 2)
|
||||
return f"areas/{domain}/{slug}.md"
|
||||
if kind in DATE_KINDS:
|
||||
return f"{folder}/{date or chorus.today()}-{slug}.md"
|
||||
return f"{folder}/{slug}.md"
|
||||
|
||||
|
||||
_KIND_RULES = [
|
||||
(r"^resources/people/", "person"),
|
||||
(r"^resources/companies/", "company"),
|
||||
(r"^resources/concepts/", "concept"),
|
||||
(r"^resources/references/", "reference"),
|
||||
(r"^resources/meetings/", "meeting"),
|
||||
(r"^projects/(active|incubating|on-hold|archived)/", "project"),
|
||||
(r"^areas/[^/]+/", "area"),
|
||||
(r"^decisions/by-date/", "decision"),
|
||||
(r"^_agent/memory/semantic/", "semantic"),
|
||||
(r"^_agent/memory/episodic/", "episodic"),
|
||||
(r"^_agent/skills/(active|archived)/", "skill"),
|
||||
]
|
||||
|
||||
|
||||
def kind_for_path(path: str) -> str | None:
|
||||
"""The entity kind a vault path belongs to (the inverse of derive_path), or None
|
||||
if the path is not an indexable entity note (journal, sessions, inbox, ...)."""
|
||||
for rx, kind in _KIND_RULES:
|
||||
if re.match(rx, path):
|
||||
return kind
|
||||
return None
|
||||
|
||||
|
||||
def empty_index() -> dict:
|
||||
return {"schema": 1, "updated": chorus.today(), "entities": {}}
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
status, body = chorus.request("GET", chorus.vault_url(INDEX_PATH))
|
||||
if status == 404:
|
||||
return empty_index()
|
||||
chorus.check(status, body, "index load")
|
||||
try:
|
||||
d = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return empty_index()
|
||||
d.setdefault("entities", {})
|
||||
return d
|
||||
|
||||
|
||||
def save(index: dict) -> None:
|
||||
index["updated"] = chorus.today()
|
||||
body = json.dumps(index, indent=2, ensure_ascii=False).encode("utf-8")
|
||||
status, b = chorus.request("PUT", chorus.vault_url(INDEX_PATH), data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
chorus.check(status, b, "index save")
|
||||
|
||||
|
||||
def _norm(s) -> str:
|
||||
return slugify(s)
|
||||
|
||||
|
||||
def resolve(index: dict, mention: str):
|
||||
"""Return (slug, entry) for a mention EXACTLY matched by slug, title, or alias — else
|
||||
(None, None). Exact-only by design: this drives capture's create-vs-update decision, so a
|
||||
loose match here would silently merge distinct entities. Fuzzy lookup is fuzzy_candidates()."""
|
||||
key = _norm(mention)
|
||||
ents = index.get("entities", {})
|
||||
if key in ents:
|
||||
return key, ents[key]
|
||||
plain = str(mention).strip().lower()
|
||||
for slug, e in ents.items():
|
||||
cands = {slug, _norm(e.get("title", "")), str(e.get("title", "")).strip().lower()}
|
||||
cands |= {_norm(a) for a in e.get("aliases", [])}
|
||||
cands |= {str(a).strip().lower() for a in e.get("aliases", [])}
|
||||
if key in cands or plain in cands:
|
||||
return slug, e
|
||||
return None, None
|
||||
|
||||
|
||||
def fuzzy_candidates(index: dict, mention: str, limit: int = 5):
|
||||
"""Advisory near-matches for when resolve() finds no exact hit. Ranks entities sharing
|
||||
>=1 *distinctive* token (>= MIN_NAME_LEN, not a common word) with the mention, so a
|
||||
shortened or expanded name — 'chorus memory' for the project 'chorus' — surfaces the existing
|
||||
note instead of vanishing. Returns [(slug, entry, score)] sorted best-first. NEVER used to
|
||||
auto-merge (that's resolve's job); only to SURFACE 'did you mean' candidates."""
|
||||
mtoks = _sig_tokens(mention)
|
||||
if not mtoks:
|
||||
return []
|
||||
scored = []
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
etoks = set()
|
||||
for name in (slug, e.get("title", ""), *e.get("aliases", [])):
|
||||
etoks |= _sig_tokens(name)
|
||||
overlap = mtoks & etoks
|
||||
if not overlap or not etoks:
|
||||
continue
|
||||
score = round(len(overlap) / min(len(mtoks), len(etoks)), 3)
|
||||
scored.append((score, len(overlap), slug, e))
|
||||
scored.sort(key=lambda x: (-x[0], -x[1], x[2]))
|
||||
return [(slug, e, score) for score, _, slug, e in scored[:limit]]
|
||||
|
||||
|
||||
def _entity_tokens(slug: str, e: dict) -> set[str]:
|
||||
toks: set[str] = set()
|
||||
for name in (slug, e.get("title", ""), *e.get("aliases", [])):
|
||||
toks |= _sig_tokens(name)
|
||||
return toks
|
||||
|
||||
|
||||
def token_df(index: dict) -> Counter:
|
||||
"""Vault-wide token document frequency: token -> how many entities carry it in their
|
||||
slug/title/aliases. A token shared by several entities ("chorus" in an chorus-centric
|
||||
vault) is a weak duplicate signal on its own; df lets the gate discount it."""
|
||||
df: Counter = Counter()
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
for t in _entity_tokens(slug, e):
|
||||
df[t] += 1
|
||||
return df
|
||||
|
||||
|
||||
def gate_candidates(index: dict, mention: str, kind: str | None = None,
|
||||
threshold: float = 0.6):
|
||||
"""The subset of fuzzy_candidates strong enough to BLOCK a create (capture's
|
||||
duplicate gate). Stricter than the advisory warning list, by two rules learned
|
||||
from live use (1.5.1):
|
||||
|
||||
* same-kind only — a cross-kind name collision (a decision titled after its
|
||||
project, a meeting named for a company) is usually intentional naming, not a
|
||||
duplicate. Cross-kind lookalikes still surface as warnings, never as a block.
|
||||
* no single-common-token blocks — score = overlap/min(|tokens|) means an entity
|
||||
whose ONE distinctive token appears in the mention scores 1.0, so any title
|
||||
containing a vault-common word ("chorus") would be blocked by every such entity.
|
||||
A lone shared token only blocks when it is unique to that entity (df == 1).
|
||||
|
||||
Returns [(slug, entry, score)] like fuzzy_candidates."""
|
||||
mtoks = _sig_tokens(mention)
|
||||
if not mtoks:
|
||||
return []
|
||||
df = token_df(index)
|
||||
out = []
|
||||
for slug, e, score in fuzzy_candidates(index, mention):
|
||||
if score < threshold:
|
||||
continue
|
||||
if kind and e.get("kind") and e["kind"] != kind:
|
||||
continue
|
||||
overlap = mtoks & _entity_tokens(slug, e)
|
||||
if len(overlap) == 1 and df[next(iter(overlap))] > 1:
|
||||
continue
|
||||
out.append((slug, e, score))
|
||||
return out
|
||||
|
||||
|
||||
def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = None,
|
||||
aliases=None) -> dict:
|
||||
ents = index.setdefault("entities", {})
|
||||
e = ents.get(slug, {})
|
||||
e["path"] = path
|
||||
e["kind"] = kind
|
||||
e["title"] = title or e.get("title") or slug
|
||||
# Keep any prior + explicit aliases, and ALWAYS fold in the safe title variants so a
|
||||
# hyphen/space/case form of the name resolves even if no one passed --aliases. Drop any
|
||||
# alias that just equals the slug (resolve already checks the slug).
|
||||
merged = set(e.get("aliases", [])) | set(aliases or []) | set(derive_aliases(e["title"]))
|
||||
merged.discard(slug)
|
||||
e["aliases"] = sorted(a for a in merged if a)
|
||||
e["last_seen"] = chorus.today()
|
||||
ents[slug] = e
|
||||
return e
|
||||
|
||||
|
||||
def name_map(index: dict) -> dict:
|
||||
"""Map every resolvable name -> path: slug, normalized title, basename, aliases.
|
||||
Used to resolve `[[wikilinks]]` and entity mentions back to vault paths."""
|
||||
m: dict[str, str] = {}
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
path = e.get("path")
|
||||
if not path:
|
||||
continue
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
base = base[:-3] if base.endswith(".md") else base
|
||||
keys = {slug, _norm(e.get("title", "")), _norm(base)}
|
||||
keys |= {_norm(a) for a in e.get("aliases", [])}
|
||||
for k in keys:
|
||||
if k:
|
||||
m.setdefault(k, path)
|
||||
return m
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_links.py — cross-link primitives.
|
||||
|
||||
Parse and add `## Related` wikilinks, and create **bidirectional** links between
|
||||
notes (read-modify-write, idempotent). Links use path-style targets without the
|
||||
`.md` extension (e.g. `[[resources/people/bob-smith]]`), which are unambiguous
|
||||
and resolve in Obsidian. Network goes through chorus.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
|
||||
WIKILINK = re.compile(r"\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]")
|
||||
|
||||
|
||||
def first_h1(text: str) -> str:
|
||||
for line in text.splitlines():
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()
|
||||
return ""
|
||||
|
||||
|
||||
def all_wikilinks(text: str) -> list[str]:
|
||||
return [m.group(1).strip() for m in WIKILINK.finditer(text)]
|
||||
|
||||
|
||||
def related_targets(text: str) -> list[str]:
|
||||
"""Wikilink targets under the `## Related` heading only."""
|
||||
out, cap = [], False
|
||||
for line in text.splitlines():
|
||||
if line.strip().lower() == "## related":
|
||||
cap = True
|
||||
continue
|
||||
if cap and line.startswith("## "):
|
||||
break
|
||||
if cap:
|
||||
out += [m.group(1).strip() for m in WIKILINK.finditer(line)]
|
||||
return out
|
||||
|
||||
|
||||
def source_notes(text: str) -> list[str]:
|
||||
"""Plain relative paths listed in the frontmatter `source_notes:` field."""
|
||||
if not text.startswith("---"):
|
||||
return []
|
||||
end = text.find("\n---", 3)
|
||||
fm = text[:end] if end != -1 else text
|
||||
m = re.search(r"(?m)^source_notes:\s*\[(.*?)\]", fm)
|
||||
if not m:
|
||||
return []
|
||||
return [p.strip().strip('"').strip("'") for p in m.group(1).split(",") if p.strip()]
|
||||
|
||||
|
||||
def link_token(path: str) -> str:
|
||||
"""Path-style wikilink target without the .md extension."""
|
||||
return path[:-3] if path.endswith(".md") else path
|
||||
|
||||
|
||||
def add_related(text: str, target_path: str):
|
||||
"""Return (new_text, changed): ensure `- [[<target>]]` exists under ## Related."""
|
||||
token = link_token(target_path)
|
||||
if token in related_targets(text):
|
||||
return text, False
|
||||
bullet = f"- [[{token}]]"
|
||||
trailing_nl = text.endswith("\n")
|
||||
lines = text.splitlines()
|
||||
idx = next((i for i, l in enumerate(lines) if l.strip().lower() == "## related"), None)
|
||||
if idx is None:
|
||||
sep = "" if trailing_nl else "\n"
|
||||
return text + f"{sep}\n## Related\n{bullet}\n", True
|
||||
# insert after the section's existing content (before the next heading / EOF)
|
||||
j = idx + 1
|
||||
while j < len(lines) and not lines[j].startswith("## "):
|
||||
j += 1
|
||||
k = j
|
||||
while k > idx + 1 and not lines[k - 1].strip():
|
||||
k -= 1
|
||||
lines.insert(k, bullet)
|
||||
return "\n".join(lines) + ("\n" if trailing_nl else ""), True
|
||||
|
||||
|
||||
def get_text(path: str) -> str | None:
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
chorus.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def put_text(path: str, text: str) -> None:
|
||||
status, body = chorus.request("PUT", chorus.vault_url(path), data=text.encode("utf-8"),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
chorus.check(status, body, f"put {path}")
|
||||
|
||||
|
||||
def add_one(path: str, target_path: str) -> bool:
|
||||
"""Add [[target]] under path's ## Related. Returns True if the file changed."""
|
||||
text = get_text(path)
|
||||
if text is None:
|
||||
return False
|
||||
new, changed = add_related(text, target_path)
|
||||
if changed:
|
||||
put_text(path, new)
|
||||
return changed
|
||||
|
||||
|
||||
def link_bidirectional(a_path: str, b_path: str):
|
||||
"""Add A<->B reciprocal links. Returns (a_changed, b_changed)."""
|
||||
if a_path == b_path:
|
||||
return (False, False)
|
||||
return (add_one(a_path, b_path), add_one(b_path, a_path))
|
||||
@@ -0,0 +1,347 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_ops.py — high-level CHORUS operations layered on the client + index + links.
|
||||
|
||||
resolve — map a mention to its canonical note path (or where to create it)
|
||||
recall — search, then expand one hop along links/source_notes for connected context
|
||||
link — create bidirectional `## Related` links between two notes
|
||||
capture — one-call write: route + canonical frontmatter + index + auto-link + agent-log
|
||||
|
||||
Network goes through chorus.py; routing/aliases through chorus_index; links through chorus_links.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
import chorus_index as idx_mod # noqa: E402
|
||||
import chorus_links as links # noqa: E402
|
||||
|
||||
# A fuzzy candidate scoring at/above this blocks `capture` from creating what is very
|
||||
# likely a duplicate (require --force to create anyway, or --merge-into to update the
|
||||
# existing entity). Below the gate, candidates are surfaced as a warning only.
|
||||
DUP_GATE = float(os.environ.get("CHORUS_DUP_GATE", "0.6"))
|
||||
|
||||
# Existing-entity capture appends a dated bullet under this per-kind heading.
|
||||
LOG_HEADING = {
|
||||
"project": "Session History", "person": "Log", "company": "Log",
|
||||
"semantic": "Observations", "area": "Log", "concept": "Notes",
|
||||
"reference": "Notes", "meeting": "Notes", "decision": "Notes",
|
||||
"skill": "Notes", "episodic": "Notes", "working": "Notes",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- resolve -----
|
||||
def resolve(mention: str) -> int:
|
||||
index = idx_mod.load()
|
||||
slug, e = idx_mod.resolve(index, mention)
|
||||
if e:
|
||||
print(json.dumps({"match": True, "slug": slug, **e}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
# No exact match — surface fuzzy candidates so a shortened/expanded name (e.g. "chorus
|
||||
# memory" for the project "chorus") reveals the existing note instead of looking absent.
|
||||
cands = idx_mod.fuzzy_candidates(index, mention)
|
||||
out = {"match": False, "mention": mention, "suggest_slug": idx_mod.slugify(mention)}
|
||||
if cands:
|
||||
out["candidates"] = [{"slug": s, "path": c.get("path"), "title": c.get("title"),
|
||||
"kind": c.get("kind"), "score": sc} for s, c, sc in cands]
|
||||
out["note"] = ("no exact match, but similar entities exist (see candidates) — check "
|
||||
"these before creating a new note; reuse the right path or `chorus.py link`.")
|
||||
else:
|
||||
out["note"] = "no index entry — derive a path with the right --kind and create it"
|
||||
print(json.dumps(out, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- link -----
|
||||
def link(a_path: str, b_path: str, as_json: bool = False) -> int:
|
||||
a_changed, b_changed = links.link_bidirectional(a_path, b_path)
|
||||
if as_json:
|
||||
import chorus_output
|
||||
env = chorus_output.envelope("link", {"a": a_path, "b": b_path,
|
||||
"a_changed": a_changed, "b_changed": b_changed})
|
||||
print(json.dumps(env, ensure_ascii=False))
|
||||
return 0
|
||||
print(f"ok: linked {a_path} <-> {b_path} "
|
||||
f"(added: {'A' if a_changed else '-'}{'B' if b_changed else '-'})")
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- recall -----
|
||||
def recall(query, limit: int = 8, as_json: bool = False) -> int:
|
||||
"""Hybrid lexical (BM25) + graph recall — implemented in chorus_recall. Kept here as
|
||||
the stable public entrypoint (chorus.py and /chorus-recall call chorus_ops.recall)."""
|
||||
import chorus_recall # lazy: only pulls the BM25 modules when recall is actually run
|
||||
return chorus_recall.recall(query, limit=limit, as_json=as_json)
|
||||
|
||||
|
||||
# ----------------------------------------------------------- agent log -------
|
||||
def ensure_daily_log(line: str) -> None:
|
||||
"""Resilient: ensure today's daily note + its `## Agent Log` heading exist, then
|
||||
idempotently append the line. Best-effort — never raises into the caller."""
|
||||
try:
|
||||
date = chorus.today()
|
||||
path = f"journal/daily/{date}.md"
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
ts, tb = chorus.request("GET", chorus.vault_url("journal/templates/daily-note-template.md"))
|
||||
tmpl = tb.decode(errors="replace") if ts == 200 else f"# {date}\n\n## Agent Log\n\n## Related\n"
|
||||
tmpl = tmpl.replace("{{date:YYYY-MM-DD}}", date).replace("{{DATE}}", date)
|
||||
chorus.request("PUT", chorus.vault_url(path), data=tmpl.encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
text = tmpl
|
||||
else:
|
||||
text = body.decode(errors="replace")
|
||||
if not re.search(r"(?m)^## Agent Log\s*$", text):
|
||||
chorus.request("POST", chorus.vault_url(path), data=b"\n\n## Agent Log\n",
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 200 and line in body.decode(errors="replace"):
|
||||
return
|
||||
chorus.request("PATCH", chorus.vault_url(path),
|
||||
data=chorus.normalize_patch_body((line + "\n").encode(), "append", "heading"),
|
||||
headers={"Operation": "append", "Target-Type": "heading",
|
||||
"Target": f"{date}::Agent Log", "Content-Type": "text/markdown"})
|
||||
except Exception as exc:
|
||||
print(f"chorus_ops: agent-log skipped ({exc})", file=sys.stderr)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- capture ---
|
||||
def _build_note(fm_type: str, status_v: str, today_s: str, title: str,
|
||||
body_text: str, sources: list[str], aliases: list[str] | None = None,
|
||||
tags: list[str] | None = None) -> str:
|
||||
src = "[" + ", ".join(json.dumps(s) for s in sources) + "]"
|
||||
fm = ["---", f"type: {fm_type}"]
|
||||
if status_v:
|
||||
fm.append(f"status: {status_v}")
|
||||
# Never scaffold an empty `tags: []` — an entity note is born complete (the caller
|
||||
# seeds the kind as a baseline tag), so the incomplete-frontmatter lint stays quiet.
|
||||
tags_v = "[" + ", ".join(tags or []) + "]"
|
||||
fm += [f"created: {today_s}", f"updated: {today_s}", f"tags: {tags_v}"]
|
||||
if aliases:
|
||||
# Obsidian-native, durable home for aliases; sweep folds these back into the index.
|
||||
fm.append("aliases: [" + ", ".join(json.dumps(a) for a in aliases) + "]")
|
||||
fm += ["agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""]
|
||||
out = "\n".join(fm)
|
||||
if body_text.strip():
|
||||
out += body_text.strip() + "\n"
|
||||
out += "\n## Related\n"
|
||||
return out
|
||||
|
||||
|
||||
def _dated_block(today_s: str, body_text: str) -> tuple[str, str]:
|
||||
"""Render a capture body as a dated log entry that PRESERVES the whole body:
|
||||
the first line rides on the bullet, every further line is indented under it as
|
||||
a markdown continuation. Returns (bullet, full_block) — the bullet alone is the
|
||||
idempotency key (same first-line-per-day semantics as before)."""
|
||||
lines = [ln.rstrip() for ln in body_text.strip().splitlines()]
|
||||
bullet = f"- {today_s}: {lines[0] if lines else '(update)'}"
|
||||
block = bullet
|
||||
for ln in lines[1:]:
|
||||
block += "\n" + (f" {ln}" if ln else "")
|
||||
return bullet, block
|
||||
|
||||
|
||||
def _append_to_existing(path: str, kind: str, today_s: str, body_text: str) -> None:
|
||||
text = links.get_text(path) or ""
|
||||
h1 = links.first_h1(text) or path.rsplit("/", 1)[-1][:-3]
|
||||
heading = LOG_HEADING.get(kind, "Notes")
|
||||
if not re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", text):
|
||||
chorus.request("POST", chorus.vault_url(path), data=f"\n\n## {heading}\n".encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
bullet, block = _dated_block(today_s, body_text)
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 200 and bullet in body.decode(errors="replace"):
|
||||
return
|
||||
chorus.request("PATCH", chorus.vault_url(path),
|
||||
data=chorus.normalize_patch_body((block + "\n").encode(), "append", "heading"),
|
||||
headers={"Operation": "append", "Target-Type": "heading",
|
||||
"Target": f"{h1}::{heading}", "Content-Type": "text/markdown"})
|
||||
chorus.cmd_fm(path, "updated", json.dumps(today_s))
|
||||
|
||||
|
||||
def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = "",
|
||||
aliases=None, sources=None, tags=None, date: str | None = None,
|
||||
domain: str = "business", inbox: bool = False, no_log: bool = False,
|
||||
as_json: bool = False, dry_run: bool = False, force: bool = False,
|
||||
merge_into: str | None = None) -> int:
|
||||
import contextlib
|
||||
import io
|
||||
import chorus_output
|
||||
import chorus_quality
|
||||
|
||||
real_stdout = sys.stdout
|
||||
|
||||
def quiet():
|
||||
# M4: in --json mode, swallow the helper "ok:" chatter so stdout is clean JSON.
|
||||
return contextlib.redirect_stdout(io.StringIO()) if as_json else contextlib.nullcontext()
|
||||
|
||||
def done(action: str, path: str, links: int = 0, ok: bool = True, dry: bool = False,
|
||||
near=None) -> int:
|
||||
if as_json:
|
||||
act = f"dry-run:{action}" if dry else action
|
||||
data = {"kind": kind, "path": path, "title": title, "links_added": links}
|
||||
if near:
|
||||
data["near_duplicates"] = near
|
||||
env = chorus_output.envelope(act, data, ok=ok)
|
||||
print(json.dumps(env, ensure_ascii=False), file=real_stdout)
|
||||
elif dry:
|
||||
print(f"would {action} {kind or '-'} -> {path}")
|
||||
# non-dry human output is the helper "ok:" lines + the summary printed by the caller.
|
||||
return 0 if ok else 1
|
||||
|
||||
body_text = chorus.read_body(file_arg).decode("utf-8", errors="replace")
|
||||
today_s = chorus.today()
|
||||
aliases = [a.strip() for a in (aliases or []) if a.strip()]
|
||||
sources = [s.strip() for s in (sources or []) if s.strip()]
|
||||
tags = [t.strip() for t in (tags or []) if t.strip()]
|
||||
|
||||
# Unknown home -> defer to the inbox (a single idempotent capture line).
|
||||
if inbox or not kind:
|
||||
if dry_run:
|
||||
return done("inbox", "inbox/captures/inbox.md", dry=True)
|
||||
line = f"- {today_s}: {title}"
|
||||
if body_text.strip():
|
||||
line += f" — {body_text.strip().splitlines()[0]}"
|
||||
with quiet():
|
||||
rc = chorus.cmd_append("inbox/captures/inbox.md", line)
|
||||
return done("inbox", "inbox/captures/inbox.md", ok=rc == 0)
|
||||
|
||||
slug = idx_mod.slugify(title)
|
||||
index = idx_mod.load()
|
||||
match_slug, existing = idx_mod.resolve(index, title)
|
||||
# --merge-into: the operator has already identified the canonical entity (e.g. after
|
||||
# a duplicate-gate stop) — route this capture as an UPDATE to it, whatever the title.
|
||||
if merge_into and not existing:
|
||||
match_slug, existing = idx_mod.resolve(index, merge_into)
|
||||
if not existing:
|
||||
print(f"chorus_ops: --merge-into '{merge_into}' matches no entity in the index "
|
||||
f"(try `resolve` first)", file=sys.stderr)
|
||||
return 2
|
||||
existing_reachable = bool(existing and chorus.request("GET", chorus.vault_url(existing["path"]))[0] == 200)
|
||||
|
||||
# Pre-write duplicate gate: a strong fuzzy candidate means this title is very likely
|
||||
# an existing entity under a different name. STOP before creating the duplicate —
|
||||
# after the fact, the warning arrives too late (the parallel note already exists).
|
||||
# gate_candidates applies the 1.5.1 precision rules (same-kind only; a lone shared
|
||||
# token blocks only when unique to that entity) so vault-common words can't gate.
|
||||
gate_hits = []
|
||||
if not existing_reachable and not force and not dry_run:
|
||||
gate_hits = [{"slug": s, "path": c.get("path"), "title": c.get("title"),
|
||||
"kind": c.get("kind"), "score": sc}
|
||||
for s, c, sc in idx_mod.gate_candidates(index, title, kind=kind,
|
||||
threshold=DUP_GATE)]
|
||||
if gate_hits:
|
||||
if as_json:
|
||||
env = chorus_output.envelope("duplicate-gate", {
|
||||
"kind": kind, "title": title, "candidates": gate_hits,
|
||||
"note": "likely duplicate — re-run with --merge-into <slug> to update the "
|
||||
"existing entity, or --force to create anyway"}, ok=False)
|
||||
print(json.dumps(env, ensure_ascii=False), file=real_stdout)
|
||||
else:
|
||||
print(f"STOP: '{title}' likely already exists — not creating a duplicate.",
|
||||
file=real_stdout)
|
||||
for c in gate_hits:
|
||||
print(f" candidate: {c['slug']} -> {c['path']} (score {c['score']})",
|
||||
file=real_stdout)
|
||||
print(" re-run with --merge-into <slug> to update the existing entity, "
|
||||
"or --force to create anyway.", file=real_stdout)
|
||||
return 76 # distinct exit: duplicate gate (cf. 75 lock, 78 config)
|
||||
|
||||
if dry_run:
|
||||
if existing_reachable:
|
||||
return done("update", existing["path"], dry=True)
|
||||
s2 = chorus_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
|
||||
near = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
|
||||
plan = done("create", idx_mod.derive_path(kind, s2, date=date, domain=domain),
|
||||
dry=True, near=near)
|
||||
if (not as_json and not force
|
||||
and idx_mod.gate_candidates(index, title, kind=kind, threshold=DUP_GATE)):
|
||||
# (in --json mode the near_duplicates field carries this; keep stdout clean)
|
||||
print("note: a real run would STOP at the duplicate gate — use --merge-into "
|
||||
"or --force.", file=real_stdout)
|
||||
return plan
|
||||
|
||||
near_dupes: list[str] = []
|
||||
index_title = title # title to record in the index entry
|
||||
index_aliases = list(aliases)
|
||||
with quiet():
|
||||
if existing_reachable:
|
||||
# Reuse the matched entity's CANONICAL slug — never re-slug from the mention, or
|
||||
# two index slugs end up pointing at one note. Keep its title; learn the mention
|
||||
# as an alias when it's a new distinctive form, so this name resolves next time.
|
||||
slug = match_slug or slug
|
||||
path = existing["path"]
|
||||
kind = existing.get("kind", kind)
|
||||
index_title = existing.get("title") or title
|
||||
known = {idx_mod._norm(a) for a in existing.get("aliases", [])} | {match_slug}
|
||||
if idx_mod.slugify(title) not in known and len(title.strip()) >= chorus_quality.MIN_NAME_LEN:
|
||||
index_aliases.append(title)
|
||||
_append_to_existing(path, kind, today_s, body_text)
|
||||
action = "updated"
|
||||
else:
|
||||
if not status_v:
|
||||
# Every entity note is born with a status — a kind-appropriate default
|
||||
# (KIND_STATUS) instead of the old project-only special case. Missing
|
||||
# status was the root cause of the frontmatter-completeness drift.
|
||||
status_v = idx_mod.KIND_STATUS.get(kind, "active")
|
||||
# Surface (don't silently create alongside) an existing entity this resembles —
|
||||
# the duplication trap when a shortened name didn't resolve exactly. (Strong
|
||||
# candidates already stopped at the duplicate gate above; these are the weak
|
||||
# ones, surfaced as a warning.)
|
||||
near_dupes = [c.get("path") for _, c, _ in idx_mod.fuzzy_candidates(index, title)][:3]
|
||||
# M2: don't let a 40-char-truncated slug silently collide with a different
|
||||
# entity already in the index — disambiguate to slug-2, slug-3, ...
|
||||
slug = chorus_quality.safe_slug(set(index.get("entities", {}).keys()), slug)
|
||||
path = idx_mod.derive_path(kind, slug, date=date, domain=domain)
|
||||
note_aliases = sorted((set(index_aliases) | set(idx_mod.derive_aliases(title))) - {slug})
|
||||
note_tags = sorted({kind, *tags}) # baseline `- <kind>` tag; --tags enriches
|
||||
note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title,
|
||||
body_text, sources, note_aliases, tags=note_tags)
|
||||
chorus.cmd_put(path, chorus.temp_file(note.encode()))
|
||||
action = "created"
|
||||
|
||||
# H3: the entity-index write is the race the review flagged — a bare load->save lets
|
||||
# two concurrent captures clobber each other's entry. Route it through a lock-guarded,
|
||||
# fresh-re-read transaction. Returns the fresh index (used by auto-link below).
|
||||
import chorus_concurrency
|
||||
index = chorus_concurrency.atomic_index_update(
|
||||
lambda idx: idx_mod.upsert(idx, slug, path, kind, index_title, index_aliases))
|
||||
|
||||
# auto-link: any other known entity CONFIDENTLY named in the body (M2: is_confident_link
|
||||
# rejects short/common tokens so a 3-char alias or a word like "API" can't link everywhere).
|
||||
linked = 0
|
||||
for s2, e2 in index.get("entities", {}).items():
|
||||
if e2.get("path") in (None, path):
|
||||
continue
|
||||
names = [n for n in [e2.get("title", ""), s2, *e2.get("aliases", [])] if n]
|
||||
if any(chorus_quality.is_confident_link(n, body_text) for n in names):
|
||||
ca, cb = links.link_bidirectional(path, e2["path"])
|
||||
linked += int(ca or cb)
|
||||
|
||||
# Keep the BM25 recall index current for this note (best-effort; lock-guarded inside
|
||||
# update_note, so it can't clobber a concurrent writer either).
|
||||
try:
|
||||
import chorus_recall
|
||||
chorus_recall.update_note(path, links.get_text(path) or "")
|
||||
except Exception as exc: # never let recall-index upkeep fail a capture
|
||||
print(f"chorus_ops: recall-index update skipped ({exc})", file=sys.stderr)
|
||||
|
||||
if not no_log:
|
||||
ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]")
|
||||
|
||||
if as_json:
|
||||
done(action, path, links=linked, near=near_dupes)
|
||||
else:
|
||||
print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else ""))
|
||||
if near_dupes:
|
||||
kind_word = "entity" if len(near_dupes) == 1 else "entities"
|
||||
print(f"WARNING: similar existing {kind_word} ({', '.join(near_dupes)}) — if this is "
|
||||
f"the same thing, merge or `chorus.py link` instead of keeping a duplicate.",
|
||||
file=real_stdout)
|
||||
return 0
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_output.py — M4: machine-readable output + dry-run plumbing. [v1.0 SCAFFOLD — implemented]
|
||||
|
||||
The high-level ops (capture/recall/resolve/scope) print prose, forcing the agent to
|
||||
re-parse free text to act on a result. This provides a single JSON envelope and a
|
||||
small dry-run helper so every op can emit a structured result and preview writes.
|
||||
|
||||
INTEGRATION (merge step): thread a global `--json` / `--dry-run` flag through chorus.py's
|
||||
parser and have each cmd_* return its data dict; emit() it. In --dry-run, the write
|
||||
verbs print the envelope with action="dry-run" and perform no network mutation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
|
||||
def envelope(action: str, data: dict | None = None, ok: bool = True,
|
||||
error: str | None = None) -> dict:
|
||||
"""Canonical result shape for every high-level op."""
|
||||
env = {"ok": ok, "action": action}
|
||||
if data:
|
||||
env.update(data)
|
||||
if error:
|
||||
env["error"] = error
|
||||
return env
|
||||
|
||||
|
||||
def emit(env: dict, as_json: bool) -> None:
|
||||
"""Print either the JSON envelope (machine mode) or a one-line human summary."""
|
||||
if as_json:
|
||||
print(json.dumps(env, ensure_ascii=False))
|
||||
return
|
||||
if not env.get("ok"):
|
||||
print(f"error: {env.get('error', 'failed')}", file=sys.stderr)
|
||||
return
|
||||
bits = [env.get("action", "ok")]
|
||||
if env.get("path"):
|
||||
bits.append(str(env["path"]))
|
||||
if env.get("links_added"):
|
||||
bits.append(f"+{env['links_added']} links")
|
||||
print("ok: " + " ".join(bits))
|
||||
|
||||
|
||||
def dry_run_envelope(action: str, data: dict | None = None) -> dict:
|
||||
"""Result for a previewed-but-not-executed write."""
|
||||
return envelope(f"dry-run:{action}", data, ok=True)
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_quality.py — M2: keep the graph (which recall depends on) clean.
|
||||
[v1.0 SCAFFOLD — helpers implemented; integration into chorus_links/chorus_index is the merge step]
|
||||
|
||||
Two graph-polluting bugs today:
|
||||
1. Auto-link fires on any indexed entity whose name/alias is >=3 chars and word-matches
|
||||
the body (chorus_ops.py:236). A concept titled "API" or a 2-3 char alias links to every
|
||||
note that happens to mention the word -> noisy, wrong edges.
|
||||
2. slugify truncates to 40 chars (chorus_index.py:58) with NO collision check, so two
|
||||
distinct long titles can resolve to the same slug -> same path -> silent merge.
|
||||
|
||||
This module supplies the two guards. Integration:
|
||||
* chorus_ops auto-link loop -> gate each candidate on is_confident_link(name, body)
|
||||
* chorus_index.derive_path -> route the slug through safe_slug(existing_slugs, slug)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
# Tokens too short or too common to be a confident entity reference on their own.
|
||||
MIN_NAME_LEN = 4
|
||||
_COMMON = frozenset(
|
||||
"api app data note task user team work code test main repo file plan goal item "
|
||||
"the and for with from".split()
|
||||
)
|
||||
|
||||
|
||||
def is_confident_link(name: str, body: str) -> bool:
|
||||
"""True only if `name` is a strong enough signal to auto-link on its appearance in
|
||||
`body`. Rejects short names, common words, and non-matches. Multi-word names are
|
||||
trusted at lower length (a two-word phrase is rarely incidental)."""
|
||||
n = (name or "").strip()
|
||||
if not n:
|
||||
return False
|
||||
multiword = len(n.split()) > 1
|
||||
if not multiword:
|
||||
if len(n) < MIN_NAME_LEN or n.lower() in _COMMON:
|
||||
return False
|
||||
return re.search(rf"\b{re.escape(n)}\b", body or "", re.IGNORECASE) is not None
|
||||
|
||||
|
||||
def safe_slug(existing_slugs, slug: str, max_len: int = 40) -> str:
|
||||
"""Return `slug` if free, else a disambiguated `slug-2`, `slug-3`, ... that still
|
||||
fits `max_len` and does not collide with anything in `existing_slugs`."""
|
||||
existing = set(existing_slugs or ())
|
||||
if slug not in existing:
|
||||
return slug
|
||||
n = 2
|
||||
while True:
|
||||
suffix = f"-{n}"
|
||||
base = slug[: max_len - len(suffix)] if len(slug) + len(suffix) > max_len else slug
|
||||
cand = f"{base}{suffix}"
|
||||
if cand not in existing:
|
||||
return cand
|
||||
n += 1
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_queue.py — H2: offline write-ahead queue + read cache. [v1.0 — wired]
|
||||
|
||||
Today vault-unreachable => "proceed without memory" (SKILL.md), so a 502 (Obsidian not
|
||||
running — the most common real failure) loses everything said that session. This module
|
||||
makes explicit writes durable across an outage and lets cold-start `load` degrade to
|
||||
last-known context instead of nothing.
|
||||
|
||||
DESIGN:
|
||||
* Outbox : append-only NDJSON of intended writes. Replayed in order on the next
|
||||
reachable session. Replay is idempotent by construction — PUT/DELETE/PATCH-
|
||||
replace overwrite, and POST / PATCH-append are content-deduped (skip if the
|
||||
line/block is already present), the same strategy chorus.py append uses. Each
|
||||
record carries an idem_key so the SAME write is never queued twice.
|
||||
* Cache : last-known-good GET bodies, so `load` has a fallback. Written by cmd_load.
|
||||
* Location: a LOCAL state dir — it CANNOT live in the vault, because the vault is the
|
||||
thing that's down. Default ~/.chorus-memory/, override CHORUS_STATE_DIR.
|
||||
|
||||
Integration seam: `safe_request()` — the write verbs (cmd_put/post/append/patch/delete)
|
||||
call it instead of chorus.request; on an outage it enqueues and returns a synthesized
|
||||
(202, b"queued ...") so the caller reports "queued, will sync" instead of failing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
|
||||
MUTATING = {"PUT", "POST", "PATCH", "DELETE"}
|
||||
|
||||
|
||||
def state_dir() -> Path:
|
||||
# Env first (chorus_config aliases a legacy ECHO_STATE_DIR at import); the
|
||||
# default is ~/.chorus-memory, but an existing pre-fork ~/.echo-memory is
|
||||
# honored when the new dir doesn't exist yet — so queued offline writes
|
||||
# from an ECHO install aren't stranded by the rename.
|
||||
override = os.environ.get("CHORUS_STATE_DIR") or os.environ.get("ECHO_STATE_DIR")
|
||||
if override:
|
||||
return Path(override)
|
||||
new = Path.home() / ".chorus-memory"
|
||||
legacy = Path.home() / ".echo-memory"
|
||||
return legacy if (not new.exists() and legacy.exists()) else new
|
||||
|
||||
|
||||
def outbox_path() -> Path:
|
||||
return state_dir() / "outbox.ndjson"
|
||||
|
||||
|
||||
def cache_dir() -> Path:
|
||||
return state_dir() / "cache"
|
||||
|
||||
|
||||
def _ensure_dirs() -> None:
|
||||
state_dir().mkdir(parents=True, exist_ok=True)
|
||||
cache_dir().mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _key(method: str, url: str, data: bytes | None) -> str:
|
||||
h = hashlib.sha1()
|
||||
h.update(method.encode())
|
||||
h.update(b"\0")
|
||||
h.update(url.encode())
|
||||
h.update(b"\0")
|
||||
h.update(data or b"")
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- outbox
|
||||
def enqueue(method: str, url: str, data: bytes | None, headers: dict, idem_key: str | None = None) -> None:
|
||||
"""Append one intended write to the outbox (NDJSON; base64 body for binary safety).
|
||||
Dedups: if a record with the same idem_key is already pending, do nothing."""
|
||||
idem_key = idem_key or _key(method, url, data)
|
||||
if any(rec.get("idem_key") == idem_key for rec in pending()):
|
||||
return
|
||||
_ensure_dirs()
|
||||
rec = {
|
||||
"method": method.upper(), "url": url,
|
||||
"body_b64": base64.b64encode(data).decode() if data else None,
|
||||
"headers": headers or {}, "idem_key": idem_key,
|
||||
# No wall-clock: replay order is file order, not a timestamp (keeps this testable).
|
||||
}
|
||||
with outbox_path().open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def pending() -> list[dict]:
|
||||
p = outbox_path()
|
||||
if not p.exists():
|
||||
return []
|
||||
return [json.loads(ln) for ln in p.read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
|
||||
|
||||
def _rewrite(records: list[dict]) -> None:
|
||||
p = outbox_path()
|
||||
if not records:
|
||||
if p.exists():
|
||||
p.unlink()
|
||||
return
|
||||
_ensure_dirs()
|
||||
tmp = p.with_suffix(".ndjson.tmp")
|
||||
tmp.write_text("".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records), encoding="utf-8")
|
||||
os.replace(tmp, p) # atomic swap so a crash mid-flush can't corrupt the queue
|
||||
|
||||
|
||||
def _rebase(url: str) -> str:
|
||||
"""Re-point a queued URL at the CURRENT chorus.BASE. Writes are queued with the base
|
||||
that was active when they failed; on replay the vault may be back at a different base
|
||||
(e.g. an endpoint migration), so we always reconstruct the origin from chorus.BASE."""
|
||||
parts = urllib.parse.urlsplit(url)
|
||||
return chorus.BASE + parts.path + (("?" + parts.query) if parts.query else "")
|
||||
|
||||
|
||||
def _replay(rec: dict) -> str:
|
||||
"""Re-issue one queued write idempotently. Returns 'ok' (landed/already-present),
|
||||
'drop' (permanent 4xx — can never succeed), or 'retry' (still offline / 5xx)."""
|
||||
method = rec["method"]
|
||||
url = _rebase(rec["url"])
|
||||
data = base64.b64decode(rec["body_b64"]) if rec.get("body_b64") else None
|
||||
headers = rec.get("headers") or {}
|
||||
|
||||
# Append-style writes: skip if the content is already present (idempotency on replay).
|
||||
is_append = method == "POST" or (method == "PATCH" and headers.get("Operation") == "append")
|
||||
if is_append and data:
|
||||
st, cur = chorus.request("GET", url)
|
||||
if st == 0:
|
||||
return "retry" # still offline
|
||||
if st == 200 and data.decode("utf-8", "replace").strip("\n") in cur.decode("utf-8", "replace"):
|
||||
return "ok" # already landed (or a duplicate we must not re-add)
|
||||
|
||||
st, body = chorus.request(method, url, data=data, headers=headers)
|
||||
if st == 0 or 500 <= st < 600:
|
||||
return "retry"
|
||||
if 400 <= st < 500:
|
||||
print(f"chorus_queue: dropping a queued {method} {url} — permanent HTTP {st} on replay",
|
||||
file=sys.stderr)
|
||||
return "drop"
|
||||
return "ok"
|
||||
|
||||
|
||||
def flush() -> int:
|
||||
"""Replay queued writes in order; stop at the first still-unreachable one (keeping it
|
||||
and the rest). Returns the number actually replayed. Safe to call when empty."""
|
||||
items = pending()
|
||||
if not items:
|
||||
return 0
|
||||
replayed = 0
|
||||
remaining: list[dict] = []
|
||||
for i, rec in enumerate(items):
|
||||
result = _replay(rec)
|
||||
if result == "ok":
|
||||
replayed += 1
|
||||
elif result == "drop":
|
||||
continue # warned in _replay; remove from queue
|
||||
else: # retry -> still offline; keep this and everything after, in order
|
||||
remaining = items[i:]
|
||||
break
|
||||
_rewrite(remaining)
|
||||
return replayed
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- cache
|
||||
def cache_key(path: str) -> Path:
|
||||
return cache_dir() / (path.replace("/", "__") + ".cache")
|
||||
|
||||
|
||||
def cache_put(path: str, body: bytes) -> None:
|
||||
_ensure_dirs()
|
||||
cache_key(path).write_bytes(body)
|
||||
|
||||
|
||||
def cache_get(path: str) -> bytes | None:
|
||||
p = cache_key(path)
|
||||
return p.read_bytes() if p.exists() else None
|
||||
|
||||
|
||||
# ----------------------------------------------------------- integration seam
|
||||
def safe_request(method: str, url: str, data=None, headers=None, idem_key: str | None = None):
|
||||
"""Network-first wrapper around chorus.request with offline queueing for writes.
|
||||
|
||||
- success or a client error (4xx): return (status, body) as-is — 4xx is a bug, fail loud.
|
||||
- transport failure (status 0) or 5xx (after chorus.request's own retry) on a MUTATING
|
||||
verb: enqueue the write and return (202, b"queued (offline)") so the caller reports it
|
||||
as durably queued rather than failed.
|
||||
- otherwise (e.g. a failing GET): return (status, body); the read-cache fallback lives
|
||||
in cmd_load, which knows which reads are worth serving stale.
|
||||
"""
|
||||
method = method.upper()
|
||||
status, body = chorus.request(method, url, data=data, headers=headers)
|
||||
offline = status == 0 or 500 <= status < 600
|
||||
if offline and method in MUTATING:
|
||||
enqueue(method, url, data, headers or {}, idem_key)
|
||||
return 202, b"queued (offline)"
|
||||
return status, body
|
||||
@@ -0,0 +1,494 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_recall.py — H1: hybrid lexical (BM25) + graph recall. [v1.0 — wired]
|
||||
|
||||
Replaces the keyword-only recall (search/simple + fixed one-hop) with a ranked, local
|
||||
retriever that finds a note even when the query is phrased differently than the note
|
||||
was stored, then expands along the graph with a per-hop decay so the *connected*
|
||||
context is surfaced in relevance order.
|
||||
|
||||
DESIGN (bounded by the project's pure-stdlib, no-deps constraint):
|
||||
* Lexical layer : BM25 over note bodies. No embeddings / external API — BM25 is a
|
||||
strong, dependency-free baseline that handles vocabulary mismatch
|
||||
far better than substring search.
|
||||
* Graph layer : from the top lexical hits, walk body wikilinks + source_notes up
|
||||
to MAX_HOPS, scoring each node `parent * GRAPH_DECAY**hop` (max over
|
||||
paths) so near, strongly-cued neighbours rank above distant ones.
|
||||
* Persistence : the BM25 postings + doc stats are a DERIVED artifact, so they live
|
||||
in the vault's machine index dir (`_agent/index/recall-index.json`),
|
||||
maintained on `capture` (update_note) and fully rebuilt by `sweep.py`
|
||||
— exactly how `entities.json` is maintained. Rebuildable => portable.
|
||||
* Resilience : if the index can't be built (vault unreachable), recall falls back
|
||||
to the server's /search/simple so it degrades instead of dying.
|
||||
|
||||
Corpus = the indexable ENTITY notes (idx_mod.kind_for_path is not None) — people,
|
||||
companies, concepts, references, meetings, projects, areas, decisions, semantic/
|
||||
episodic memory, skills — PLUS the time-series notes where conversation-derived
|
||||
context actually lives: session logs (`_agent/sessions/`) and journal notes
|
||||
(`journal/daily|weekly|monthly|quarterly|annual/`). Time-series docs carry a
|
||||
per-kind DOWN-WEIGHT so entity notes still rank first for the same lexical match.
|
||||
|
||||
RANKING (v1.5): final score = BM25 * kind_weight * freshness * status_factor —
|
||||
* kind_weight : 1.0 entity · 0.5 session · 0.4 journal (KIND_WEIGHT)
|
||||
* freshness : half-life decay on `updated:` (or the filename date), floored at
|
||||
FRESH_FLOOR so old notes are demoted, never buried
|
||||
* status : `active` boosted, `archived` demoted (STATUS_FACTOR)
|
||||
so a stale archived project no longer outranks the live one on raw term frequency.
|
||||
Doc metadata rides in the index (schema 2); a schema-1 index is discarded and
|
||||
rebuilt on the next recall/sweep.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
from collections import Counter, deque
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
import chorus_index as idx_mod # noqa: E402
|
||||
import chorus_links as links # noqa: E402
|
||||
|
||||
RECALL_INDEX_PATH = "_agent/index/recall-index.json"
|
||||
INDEX_SCHEMA = 2 # bumped: schema 2 adds per-doc meta (weight/updated/status)
|
||||
|
||||
# Tuning knobs. Defaults are standard BM25 (k1/b) + a 0.6 hop decay over <=2 hops.
|
||||
K1 = 1.5
|
||||
B = 0.75
|
||||
MAX_HOPS = 2
|
||||
GRAPH_DECAY = 0.6
|
||||
MIN_SCORE = 0.0 # relevance floor; raise to trim weak lexical hits
|
||||
|
||||
# --- rank-fusion priors -------------------------------------------------------
|
||||
# Time-series notes are in the corpus but down-weighted: they are where "what did we
|
||||
# decide three weeks ago" lives, yet the entity note should win a tie on the same terms.
|
||||
_TIME_SERIES = [
|
||||
(re.compile(r"^_agent/sessions/"), "session"),
|
||||
(re.compile(r"^journal/daily/"), "daily"),
|
||||
(re.compile(r"^journal/(weekly|monthly|quarterly|annual)/"), "rollup"),
|
||||
]
|
||||
KIND_WEIGHT = {"session": 0.5, "daily": 0.4, "rollup": 0.4}
|
||||
STATUS_FACTOR = {"active": 1.1, "on-hold": 0.9, "archived": 0.75}
|
||||
FRESH_HALF_LIFE = float(os.environ.get("CHORUS_FRESH_HALF_LIFE", "90")) # days
|
||||
FRESH_FLOOR = 0.6 # freshness multiplier never drops below this — demote, don't bury
|
||||
|
||||
_FM_UPDATED = re.compile(r"(?m)^updated:\s*[\"']?(\d{4}-\d{2}-\d{2})")
|
||||
_FM_STATUS = re.compile(r"(?m)^status:\s*[\"']?([A-Za-z-]+)")
|
||||
_PATH_DATE = re.compile(r"(\d{4}-\d{2}-\d{2})")
|
||||
|
||||
|
||||
def _series_kind(path: str) -> str | None:
|
||||
for rx, kind in _TIME_SERIES:
|
||||
if rx.match(path):
|
||||
return kind
|
||||
return None
|
||||
|
||||
|
||||
def doc_meta(path: str, raw_text: str) -> dict:
|
||||
"""Per-doc ranking priors, extracted once at index time: kind weight, last-updated
|
||||
date (frontmatter `updated:`, else the date in the filename), and `status:`."""
|
||||
kind = _series_kind(path)
|
||||
w = KIND_WEIGHT.get(kind, 1.0) if kind else 1.0
|
||||
head = raw_text[:2000]
|
||||
mu = _FM_UPDATED.search(head)
|
||||
updated = mu.group(1) if mu else None
|
||||
if updated is None:
|
||||
mp = _PATH_DATE.search(path.rsplit("/", 1)[-1])
|
||||
updated = mp.group(1) if mp else None
|
||||
ms = _FM_STATUS.search(head)
|
||||
meta = {"w": w}
|
||||
if updated:
|
||||
meta["u"] = updated
|
||||
if ms:
|
||||
meta["s"] = ms.group(1)
|
||||
return meta
|
||||
|
||||
|
||||
def freshness(updated: str | None, today_s: str) -> float:
|
||||
"""Half-life decay on document age, floored. Unknown age is neutral (1.0)."""
|
||||
if not updated:
|
||||
return 1.0
|
||||
try:
|
||||
age = (_dt.date.fromisoformat(today_s) - _dt.date.fromisoformat(updated)).days
|
||||
except ValueError:
|
||||
return 1.0
|
||||
if age <= 0:
|
||||
return 1.0
|
||||
return FRESH_FLOOR + (1.0 - FRESH_FLOOR) * (0.5 ** (age / FRESH_HALF_LIFE))
|
||||
|
||||
_WORD = re.compile(r"[a-z0-9]+")
|
||||
# Minimal English stoplist — keeps BM25 idf from being dominated by glue words.
|
||||
_STOP = frozenset(
|
||||
"the a an and or of to in is are was were be been for on at by with as it this that "
|
||||
"from into over under not no yes do does did has have had will would can could".split()
|
||||
)
|
||||
_TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
_SKIP_BASENAMES = {"README.md"}
|
||||
|
||||
|
||||
def tokenize(text: str) -> list[str]:
|
||||
"""Lowercase word tokens, stopworded. Frontmatter should be stripped by the caller."""
|
||||
return [t for t in _WORD.findall(text.lower()) if t not in _STOP and len(t) > 1]
|
||||
|
||||
|
||||
def strip_frontmatter(text: str) -> str:
|
||||
if not text.startswith("---"):
|
||||
return text
|
||||
end = text.find("\n---", 3)
|
||||
return text[end + 4:] if end != -1 else text
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- BM25 core
|
||||
class Bm25Index:
|
||||
"""A compact, serializable BM25 index with per-doc ranking meta.
|
||||
add()/remove() are idempotent per path. add() takes the RAW note text — it strips
|
||||
frontmatter itself and extracts the doc meta (weight/updated/status) in one pass."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.df: Counter[str] = Counter() # term -> #docs containing it
|
||||
self.postings: dict[str, dict[str, int]] = {} # term -> {doc_path: tf}
|
||||
self.length: dict[str, int] = {} # doc_path -> token count
|
||||
self.meta: dict[str, dict] = {} # doc_path -> {w, u, s} priors
|
||||
self.n_docs = 0
|
||||
self.avg_len = 0.0
|
||||
|
||||
def _recompute(self) -> None:
|
||||
self.n_docs = len(self.length)
|
||||
self.avg_len = (sum(self.length.values()) / self.n_docs) if self.n_docs else 0.0
|
||||
|
||||
def add(self, path: str, raw_text: str) -> None:
|
||||
if path in self.length: # re-index in place (idempotent)
|
||||
self.remove(path)
|
||||
toks = tokenize(strip_frontmatter(raw_text))
|
||||
self.length[path] = len(toks)
|
||||
self.meta[path] = doc_meta(path, raw_text)
|
||||
for term, tf in Counter(toks).items():
|
||||
self.postings.setdefault(term, {})[path] = tf
|
||||
self.df[term] += 1
|
||||
self._recompute()
|
||||
|
||||
def remove(self, path: str) -> None:
|
||||
if path not in self.length:
|
||||
return
|
||||
for term in list(self.postings.keys()):
|
||||
if path in self.postings[term]:
|
||||
del self.postings[term][path]
|
||||
self.df[term] -= 1
|
||||
if not self.postings[term]:
|
||||
del self.postings[term]
|
||||
del self.df[term]
|
||||
del self.length[path]
|
||||
self.meta.pop(path, None)
|
||||
self._recompute()
|
||||
|
||||
def _doc_factor(self, path: str, today_s: str) -> float:
|
||||
"""The rank-fusion prior: kind weight x freshness x status factor."""
|
||||
m = self.meta.get(path) or {}
|
||||
return (float(m.get("w", 1.0))
|
||||
* freshness(m.get("u"), today_s)
|
||||
* STATUS_FACTOR.get(m.get("s", ""), 1.0))
|
||||
|
||||
def score(self, query: str, limit: int = 10,
|
||||
today_s: str | None = None) -> list[tuple[str, float]]:
|
||||
today_s = today_s or chorus.today()
|
||||
scores: Counter[str] = Counter()
|
||||
for term in set(tokenize(query)):
|
||||
posting = self.postings.get(term)
|
||||
if not posting:
|
||||
continue
|
||||
idf = math.log(1 + (self.n_docs - self.df[term] + 0.5) / (self.df[term] + 0.5))
|
||||
for path, tf in posting.items():
|
||||
dl = self.length.get(path, 0)
|
||||
denom = tf + K1 * (1 - B + B * (dl / self.avg_len if self.avg_len else 1))
|
||||
scores[path] += idf * (tf * (K1 + 1) / denom if denom else 0)
|
||||
fused = {p: s * self._doc_factor(p, today_s) for p, s in scores.items()}
|
||||
ranked = sorted(fused.items(), key=lambda kv: -kv[1])[:limit]
|
||||
return [(p, s) for p, s in ranked if s > MIN_SCORE]
|
||||
|
||||
# ---- serialization (compact; rebuildable, so the format can change freely) ----
|
||||
def to_json(self) -> dict:
|
||||
return {"schema": INDEX_SCHEMA, "n_docs": self.n_docs, "avg_len": self.avg_len,
|
||||
"length": self.length, "postings": self.postings, "meta": self.meta}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, d: dict) -> "Bm25Index":
|
||||
if d.get("schema") != INDEX_SCHEMA:
|
||||
return cls() # older/newer format -> empty; recall's cold path rebuilds
|
||||
ix = cls()
|
||||
ix.length = d.get("length", {})
|
||||
ix.postings = d.get("postings", {})
|
||||
ix.meta = d.get("meta", {})
|
||||
ix.n_docs = d.get("n_docs", len(ix.length))
|
||||
ix.avg_len = d.get("avg_len", 0.0)
|
||||
ix.df = Counter({term: len(posting) for term, posting in ix.postings.items()})
|
||||
return ix
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- vault I/O
|
||||
def _get(path: str) -> str | None:
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
chorus.check(status, body, f"recall get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def _list_dir(path: str):
|
||||
p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/")
|
||||
body = _get(p)
|
||||
if body is None:
|
||||
return [], []
|
||||
try:
|
||||
j = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [], []
|
||||
entries = list(j.get("files", [])) + list(j.get("folders", []))
|
||||
files = [e for e in entries if not e.endswith("/")]
|
||||
folders = [e[:-1] for e in entries if e.endswith("/")]
|
||||
return files, folders
|
||||
|
||||
|
||||
def _walk(prefix: str = ""):
|
||||
files, folders = _list_dir(prefix)
|
||||
for f in files:
|
||||
yield prefix + f
|
||||
for d in folders:
|
||||
yield from _walk(f"{prefix}{d}/")
|
||||
|
||||
|
||||
def _indexable(path: str) -> bool:
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
if (not path.endswith(".md") or base in _SKIP_BASENAMES
|
||||
or _TEMPLATE_RE.search(path)):
|
||||
return False
|
||||
# Entity notes (the durable graph) + time-series notes (sessions, journal) — the
|
||||
# latter join the corpus down-weighted so past decisions/discussions are findable.
|
||||
return idx_mod.kind_for_path(path) is not None or _series_kind(path) is not None
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- persistence
|
||||
def load_index() -> Bm25Index:
|
||||
status, body = chorus.request("GET", chorus.vault_url(RECALL_INDEX_PATH))
|
||||
if status == 404:
|
||||
return Bm25Index()
|
||||
chorus.check(status, body, "recall-index load")
|
||||
try:
|
||||
return Bm25Index.from_json(json.loads(body))
|
||||
except (json.JSONDecodeError, KeyError, TypeError):
|
||||
return Bm25Index()
|
||||
|
||||
|
||||
def save_index(ix: Bm25Index) -> None:
|
||||
body = json.dumps(ix.to_json(), ensure_ascii=False).encode("utf-8")
|
||||
status, b = chorus.request("PUT", chorus.vault_url(RECALL_INDEX_PATH), data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
chorus.check(status, b, "recall-index save")
|
||||
|
||||
|
||||
def rebuild(prefetched: dict | None = None) -> Bm25Index:
|
||||
"""Full rebuild from every indexable entity note. Called by sweep.py and as the
|
||||
cold-cache path inside recall(). Saves and returns the index.
|
||||
|
||||
`prefetched` (a {path: text} map, e.g. from chorus.read_many) lets sweep.py reuse the
|
||||
content it already pulled instead of walking and re-fetching the whole vault again."""
|
||||
ix = Bm25Index()
|
||||
if prefetched is not None:
|
||||
items = ((p, t) for p, t in prefetched.items() if _indexable(p))
|
||||
else:
|
||||
items = ((p, _get(p)) for p in _walk() if _indexable(p))
|
||||
for path, text in items:
|
||||
if text is not None:
|
||||
ix.add(path, text) # add() strips frontmatter + extracts doc meta itself
|
||||
save_index(ix)
|
||||
return ix
|
||||
|
||||
|
||||
def update_note(path: str, text: str) -> None:
|
||||
"""Best-effort incremental upkeep for a single note (mirrors how entities.json is
|
||||
maintained on capture). Never raises into the caller. The load->save is wrapped in the
|
||||
advisory lock with a fresh re-read (H3), so concurrent captures can't clobber the
|
||||
recall index either."""
|
||||
try:
|
||||
if not _indexable(path):
|
||||
return # only entity notes are part of the recall corpus
|
||||
import chorus_concurrency
|
||||
with chorus_concurrency.vault_lock():
|
||||
ix = load_index() # fresh read inside the lock
|
||||
ix.add(path, text) # raw text: add() strips + extracts meta
|
||||
save_index(ix)
|
||||
except Exception as exc: # noqa: BLE001 — upkeep must never fail a capture
|
||||
print(f"chorus_recall: index update skipped ({exc})", file=sys.stderr)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- graph fusion
|
||||
def _resolve_target(target: str, nmap: dict) -> str | None:
|
||||
target = target.strip()
|
||||
if not target:
|
||||
return None
|
||||
if "/" in target:
|
||||
return target if target.endswith(".md") else target + ".md"
|
||||
return nmap.get(idx_mod.slugify(target))
|
||||
|
||||
|
||||
def expand_graph(seeds, nmap: dict, base_scores: dict, max_hops: int = MAX_HOPS):
|
||||
"""BFS from seeds over body wikilinks + source_notes, scoring each neighbour
|
||||
`parent_score * GRAPH_DECAY**hop` (kept as the max over reaching paths). Returns a
|
||||
score-sorted list of (path, (score, via)) excluding the seeds themselves."""
|
||||
score_of = dict(base_scores)
|
||||
seen = set(seeds)
|
||||
results: dict[str, tuple[float, str]] = {}
|
||||
# Breadth-first by HOP so each frontier's note bodies can be fetched concurrently
|
||||
# (chorus.read_many) instead of one blocking GET per node. Scoring is unchanged: a
|
||||
# neighbour keeps the max decayed score over the paths that reach it, and non-seed
|
||||
# parents score 1.0 exactly as the serial deque version did.
|
||||
frontier = list(seeds)
|
||||
for hop in range(max_hops):
|
||||
if not frontier:
|
||||
break
|
||||
texts = chorus.read_many(frontier)
|
||||
next_frontier: list[str] = []
|
||||
for path in frontier:
|
||||
text = texts.get(path)
|
||||
if text is None:
|
||||
continue
|
||||
body = strip_frontmatter(text)
|
||||
targets = set(links.all_wikilinks(body)) | set(links.source_notes(text))
|
||||
parent = score_of.get(path, 1.0)
|
||||
decayed = parent * (GRAPH_DECAY ** (hop + 1))
|
||||
for t in targets:
|
||||
tp = _resolve_target(t, nmap)
|
||||
if not tp or tp in seeds:
|
||||
continue
|
||||
if decayed > results.get(tp, (0.0, ""))[0]:
|
||||
results[tp] = (decayed, path)
|
||||
if tp not in seen:
|
||||
seen.add(tp)
|
||||
next_frontier.append(tp)
|
||||
frontier = next_frontier
|
||||
return sorted(results.items(), key=lambda kv: -kv[1][0])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- presentation
|
||||
def _doc_summary(path: str, text: str) -> dict:
|
||||
"""Structured per-note summary shared by the human brief and --json output."""
|
||||
out: dict = {"path": path}
|
||||
mtype = re.search(r"(?m)^type:\s*(.+)$", text)
|
||||
if mtype:
|
||||
out["type"] = mtype.group(1).strip()
|
||||
mu = _FM_UPDATED.search(text[:2000])
|
||||
if mu:
|
||||
out["updated"] = mu.group(1)
|
||||
ms = _FM_STATUS.search(text[:2000])
|
||||
if ms:
|
||||
out["status"] = ms.group(1)
|
||||
body = strip_frontmatter(text)
|
||||
status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body)
|
||||
if status and status.group(1).strip():
|
||||
out["excerpt"] = status.group(1).strip()[:400]
|
||||
else:
|
||||
kept = [ln[:200] for ln in body.splitlines()
|
||||
if ln.strip() and not ln.startswith("# ")][:6]
|
||||
out["excerpt"] = "\n".join(kept)
|
||||
return out
|
||||
|
||||
|
||||
def _brief(path: str, score: float | None = None, via: str | None = None) -> None:
|
||||
text = links.get_text(path)
|
||||
if text is None:
|
||||
return
|
||||
info = _doc_summary(path, text)
|
||||
head = f"\n### {path}"
|
||||
if score is not None:
|
||||
head += f" (score {score:.2f})"
|
||||
if via:
|
||||
head += f" (via {via})"
|
||||
print(head)
|
||||
stamps = []
|
||||
if info.get("type"):
|
||||
stamps.append(f"type: {info['type']}")
|
||||
if info.get("updated"):
|
||||
stamps.append(f"updated: {info['updated']}")
|
||||
if info.get("status"):
|
||||
stamps.append(f"status: {info['status']}")
|
||||
if stamps:
|
||||
print("_" + " · ".join(stamps) + "_") # staleness is visible, not guessed
|
||||
if info.get("excerpt"):
|
||||
print(info["excerpt"])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- entrypoint
|
||||
def recall(query, limit: int = 8, as_json: bool = False) -> int:
|
||||
q = " ".join(query) if isinstance(query, list) else query
|
||||
today_s = chorus.today()
|
||||
index = idx_mod.load()
|
||||
nmap = idx_mod.name_map(index)
|
||||
|
||||
# --- lexical layer (BM25); rebuild the cache on a cold miss --------------
|
||||
lexical: list[tuple[str, float]] = []
|
||||
try:
|
||||
ix = load_index()
|
||||
if ix.n_docs == 0:
|
||||
ix = rebuild()
|
||||
lexical = ix.score(q, limit=limit, today_s=today_s)
|
||||
except chorus.ChorusError as exc:
|
||||
print(f"recall: BM25 index unavailable ({exc}); falling back to server search",
|
||||
file=sys.stderr)
|
||||
|
||||
base: dict[str, float] = {p: s for p, s in lexical}
|
||||
hits: list[str] = [p for p, _ in lexical]
|
||||
|
||||
# --- entity-index seed (alias-aware exact match) -------------------------
|
||||
_, e = idx_mod.resolve(index, q)
|
||||
if e and e.get("path") and e["path"] not in base:
|
||||
hits.insert(0, e["path"])
|
||||
base[e["path"]] = max(base.values(), default=1.0)
|
||||
|
||||
# --- resilience fallback: server search when lexical found nothing -------
|
||||
if not hits:
|
||||
try:
|
||||
status, body = chorus.request(
|
||||
"POST", f"{chorus.BASE}/search/simple/?query={urllib.parse.quote_plus(q)}")
|
||||
if status == 200:
|
||||
for r in (json.loads(body) or [])[:limit]:
|
||||
fn = r.get("filename") or r.get("path")
|
||||
if fn and fn not in base:
|
||||
hits.append(fn)
|
||||
base[fn] = 0.0
|
||||
except Exception: # noqa: BLE001 — fallback only; stay quiet
|
||||
pass
|
||||
|
||||
# --- graph layer ----------------------------------------------------------
|
||||
neighbours = expand_graph(hits, nmap, base, max_hops=MAX_HOPS)
|
||||
|
||||
if as_json:
|
||||
import chorus_output
|
||||
texts = chorus.read_many(hits + [p for p, _ in neighbours[: 2 * limit]])
|
||||
primary = []
|
||||
for p in hits:
|
||||
if texts.get(p) is None:
|
||||
continue
|
||||
primary.append({**_doc_summary(p, texts[p]),
|
||||
"score": round(base.get(p, 0.0), 3)})
|
||||
linked = []
|
||||
for p, (sc, via) in neighbours[: 2 * limit]:
|
||||
if texts.get(p) is None:
|
||||
continue
|
||||
linked.append({**_doc_summary(p, texts[p]),
|
||||
"score": round(sc, 3), "via": via})
|
||||
env = chorus_output.envelope("recall", {"query": q, "primary": primary,
|
||||
"linked": linked})
|
||||
print(json.dumps(env, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
print(f"== recall: {q} ==")
|
||||
print(f"\n# Primary hits ({len(hits)})")
|
||||
for p in hits:
|
||||
_brief(p, score=base.get(p))
|
||||
print(f"\n# Linked context ({len(neighbours)})")
|
||||
for p, (sc, via) in neighbours[: 2 * limit]:
|
||||
_brief(p, score=sc, via=via)
|
||||
return 0
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_reflect.py — H5: automatic session-reflection capture. [v1.0 — wired]
|
||||
|
||||
Today memory only accrues when the agent decides to write. This makes accrual the
|
||||
default: at session end the model extracts durable items from the conversation and
|
||||
emits a PROPOSAL set; this module dedups them against the entity index, previews them,
|
||||
and applies the accepted ones via the normal `capture` path — so nothing is written
|
||||
without a go-ahead (honors the "show before large/sensitive writes" safety rule).
|
||||
|
||||
DIVISION OF LABOR:
|
||||
* Extraction is MODEL-side (only the LLM has the conversation). The agent produces a
|
||||
JSON array of proposals matching PROPOSAL_SCHEMA.
|
||||
* This script is the deterministic spine: validate -> classify (new/update/inbox vs the
|
||||
entity index) -> preview -> apply on confirm. No NL understanding lives here, so it's
|
||||
testable offline.
|
||||
|
||||
PROPOSAL_SCHEMA (one per durable item the agent wants to remember):
|
||||
{
|
||||
"title": "Bob Smith", # required
|
||||
"kind": "person", # required unless "inbox": true
|
||||
"body": "Principal at Acme; ...", # markdown body (optional)
|
||||
"aliases": ["bob", "rs"], # optional
|
||||
"tags": ["client"], # optional; the kind is always seeded as a tag
|
||||
"sources": ["_agent/sessions/..."], # optional backward links
|
||||
"date": "YYYY-MM-DD", # optional (meeting/decision kinds)
|
||||
"domain": "business", # optional (area kind)
|
||||
"inbox": false, # route to inbox when the home is unknown
|
||||
"confidence": 0.0-1.0 # agent's own confidence; gates auto-suggest
|
||||
}
|
||||
|
||||
CLI (chorus.py reflect): dry-run previews; --apply writes — matching bootstrap/migrate/sweep.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
import chorus_index as idx_mod # noqa: E402
|
||||
|
||||
MIN_CONFIDENCE = 0.6
|
||||
|
||||
|
||||
def validate(proposals: list[dict]) -> tuple[list[dict], list[str]]:
|
||||
"""Return (valid, errors). title always required; kind required unless inbox; a
|
||||
confidence below the floor is routed away (the agent should send it to the inbox)."""
|
||||
valid, errors = [], []
|
||||
for i, p in enumerate(proposals or []):
|
||||
if not str(p.get("title", "")).strip():
|
||||
errors.append(f"proposal[{i}]: missing title")
|
||||
continue
|
||||
is_inbox = bool(p.get("inbox"))
|
||||
if not is_inbox:
|
||||
kind = str(p.get("kind", "")).strip()
|
||||
if not kind:
|
||||
errors.append(f"proposal[{i}] '{p['title']}': missing kind (or set inbox:true)")
|
||||
continue
|
||||
if kind not in idx_mod.KIND_FOLDER:
|
||||
errors.append(f"proposal[{i}] '{p['title']}': unknown kind '{kind}'")
|
||||
continue
|
||||
if float(p.get("confidence", 1.0)) < MIN_CONFIDENCE:
|
||||
errors.append(f"proposal[{i}] '{p['title']}': confidence < {MIN_CONFIDENCE} — send to inbox instead")
|
||||
continue
|
||||
valid.append(p)
|
||||
return valid, errors
|
||||
|
||||
|
||||
def classify(proposals: list[dict]) -> list[dict]:
|
||||
"""Annotate each proposal with `_action` (create / update / inbox) and `_path`, by
|
||||
resolving its title against the entity index (alias-aware), so the preview shows
|
||||
create-vs-append and dedups against what's already in memory."""
|
||||
index = idx_mod.load()
|
||||
for p in proposals:
|
||||
if p.get("inbox") or not p.get("kind"):
|
||||
p["_action"], p["_path"] = "inbox", "inbox/captures/inbox.md"
|
||||
continue
|
||||
_, e = idx_mod.resolve(index, p["title"])
|
||||
if e and e.get("path"):
|
||||
p["_action"], p["_path"] = "update", e["path"]
|
||||
else:
|
||||
try:
|
||||
p["_path"] = idx_mod.derive_path(
|
||||
p["kind"], idx_mod.slugify(p["title"]),
|
||||
date=p.get("date"), domain=p.get("domain", "business"))
|
||||
p["_action"] = "create"
|
||||
except chorus.ChorusError as exc:
|
||||
p["_action"], p["_path"] = "error", str(exc)
|
||||
return proposals
|
||||
|
||||
|
||||
def preview(proposals: list[dict]) -> str:
|
||||
"""One confirmable line per proposal: action | kind | title -> target path."""
|
||||
rows = []
|
||||
for p in proposals:
|
||||
act = p.get("_action", "?")
|
||||
kind = "-" if act == "inbox" else p.get("kind", "?")
|
||||
rows.append(f" {act:7} | {kind:9} | {p['title']} -> {p.get('_path', '?')}")
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def apply(proposals: list[dict], confirm: bool = False) -> int:
|
||||
"""Validate -> classify -> preview; with confirm=True, apply each via chorus_ops.capture
|
||||
(which routes/indexes/links/logs, each self-lock-guarded). Without confirm it is a
|
||||
dry-run that writes nothing — the preview IS the confirmation step."""
|
||||
valid, errors = validate(proposals)
|
||||
for e in errors:
|
||||
print(f"skip: {e}", file=sys.stderr)
|
||||
if not valid:
|
||||
print("reflect: no valid proposals to apply.")
|
||||
return 0
|
||||
|
||||
classify(valid)
|
||||
counts = {a: sum(1 for p in valid if p.get("_action") == a) for a in ("create", "update", "inbox", "error")}
|
||||
print(f"reflect: {len(valid)} proposal(s) — "
|
||||
f"{counts['create']} new, {counts['update']} update, {counts['inbox']} inbox"
|
||||
+ (f", {counts['error']} error" if counts["error"] else ""))
|
||||
print(preview(valid))
|
||||
|
||||
if not confirm:
|
||||
print("\nreflect: dry-run — re-run with --apply to write these to memory.")
|
||||
return 0
|
||||
|
||||
import chorus_ops # lazy: the apply path pulls in the capture/index/link stack
|
||||
applied = gated = 0
|
||||
for p in valid:
|
||||
if p.get("_action") == "error":
|
||||
continue
|
||||
bodyfile = chorus.temp_file((p.get("body") or "").encode()) if p.get("body") else None
|
||||
rc = chorus_ops.capture(
|
||||
p.get("kind"), p["title"], bodyfile,
|
||||
aliases=p.get("aliases") or [], sources=p.get("sources") or [],
|
||||
tags=p.get("tags") or [],
|
||||
date=p.get("date"), domain=p.get("domain", "business"),
|
||||
inbox=bool(p.get("inbox")) or not p.get("kind"))
|
||||
applied += 1 if rc == 0 else 0
|
||||
gated += 1 if rc == 76 else 0
|
||||
print(f"reflect: applied {applied}/{len(valid)} proposal(s)."
|
||||
+ (f" {gated} stopped at the duplicate gate — re-propose with the existing "
|
||||
f"entity's title (or capture --merge-into <slug>)." if gated else ""))
|
||||
return 0
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chorus_triage.py — one-tap inbox triage, built on the reflect pipeline.
|
||||
|
||||
Triage used to be pure prose: the agent hand-routed each inbox line with individual
|
||||
PATCH/PUT calls and hand-wrote the processing log — which is why the inbox rots.
|
||||
But the machinery it needs already exists in chorus_reflect (validate -> classify
|
||||
against the entity index -> preview -> apply via capture). This module reuses it
|
||||
and adds the two triage-specific pieces:
|
||||
|
||||
* `list_inbox` — parse `inbox/captures/inbox.md` into structured capture lines
|
||||
(date, text, age in days) so the model builds proposals from
|
||||
data, not by re-reading prose;
|
||||
* `apply` — same contract as reflect (dry-run unless confirm), plus an
|
||||
audit line in `inbox/processing-log/YYYY-MM-DD.md` for every
|
||||
routed item. Originals are NEVER deleted (operating contract:
|
||||
the processing log is the audit trail, deletion is explicit).
|
||||
|
||||
Proposals are reflect's PROPOSAL_SCHEMA plus an optional `"line"` — the original
|
||||
inbox line, echoed into the processing log so the audit trail maps 1:1.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import chorus # noqa: E402
|
||||
|
||||
INBOX_PATH = "inbox/captures/inbox.md"
|
||||
LOG_DIR = "inbox/processing-log"
|
||||
|
||||
_CAPTURE_LINE = re.compile(r"^\s*-\s*(\d{4}-\d{2}-\d{2})\s*:?\s*(.*\S)\s*$")
|
||||
|
||||
|
||||
def parse_inbox(text: str) -> list[dict]:
|
||||
"""Dated capture bullets -> [{line, date, text, age_days}] (age from CHORUS_TODAY)."""
|
||||
import datetime as dt
|
||||
try:
|
||||
today = dt.date.fromisoformat(chorus.today())
|
||||
except ValueError:
|
||||
today = dt.date.today()
|
||||
items = []
|
||||
for raw_line in text.splitlines():
|
||||
m = _CAPTURE_LINE.match(raw_line)
|
||||
if not m:
|
||||
continue
|
||||
try:
|
||||
age = (today - dt.date.fromisoformat(m.group(1))).days
|
||||
except ValueError:
|
||||
age = None
|
||||
items.append({"line": raw_line.strip(), "date": m.group(1),
|
||||
"text": m.group(2), "age_days": age})
|
||||
return items
|
||||
|
||||
|
||||
def list_inbox(as_json: bool = False) -> int:
|
||||
status, body = chorus.request("GET", chorus.vault_url(INBOX_PATH))
|
||||
if status == 404:
|
||||
items = []
|
||||
else:
|
||||
chorus.check(status, body, f"triage list {INBOX_PATH}")
|
||||
items = parse_inbox(body.decode(errors="replace"))
|
||||
if as_json:
|
||||
import chorus_output
|
||||
env = chorus_output.envelope("triage-list", {"path": INBOX_PATH, "items": items,
|
||||
"count": len(items)})
|
||||
print(json.dumps(env, ensure_ascii=False))
|
||||
return 0
|
||||
if not items:
|
||||
print("triage: inbox is empty — nothing to route.")
|
||||
return 0
|
||||
print(f"triage: {len(items)} capture(s) in {INBOX_PATH}")
|
||||
for it in items:
|
||||
age = f"{it['age_days']}d" if it["age_days"] is not None else "?"
|
||||
print(f" [{age:>4}] {it['date']}: {it['text']}")
|
||||
print("\nBuild a proposals JSON (reflect schema + optional \"line\") and run "
|
||||
"`chorus.py triage <file>` to preview, `--apply` to route.")
|
||||
return 0
|
||||
|
||||
|
||||
def apply(proposals: list[dict], confirm: bool = False) -> int:
|
||||
"""Validate -> classify -> preview; with confirm, route each via capture AND write
|
||||
the processing-log audit line. Mirrors chorus_reflect.apply's contract exactly."""
|
||||
import chorus_reflect
|
||||
valid, errors = chorus_reflect.validate(proposals)
|
||||
for e in errors:
|
||||
print(f"skip: {e}", file=sys.stderr)
|
||||
if not valid:
|
||||
print("triage: no valid proposals to route.")
|
||||
return 0
|
||||
|
||||
chorus_reflect.classify(valid)
|
||||
counts = {a: sum(1 for p in valid if p.get("_action") == a)
|
||||
for a in ("create", "update", "inbox", "error")}
|
||||
print(f"triage: {len(valid)} proposal(s) — "
|
||||
f"{counts['create']} new, {counts['update']} update, {counts['inbox']} stay-in-inbox"
|
||||
+ (f", {counts['error']} error" if counts["error"] else ""))
|
||||
print(chorus_reflect.preview(valid))
|
||||
|
||||
if not confirm:
|
||||
print("\ntriage: dry-run — re-run with --apply to route these and log the moves.")
|
||||
return 0
|
||||
|
||||
import chorus_ops
|
||||
today_s = chorus.today()
|
||||
applied = gated = 0
|
||||
for p in valid:
|
||||
if p.get("_action") == "error":
|
||||
continue
|
||||
if p.get("_action") == "inbox":
|
||||
continue # routing an inbox line back to the inbox is a no-op, not a move
|
||||
bodyfile = chorus.temp_file((p.get("body") or "").encode()) if p.get("body") else None
|
||||
rc = chorus_ops.capture(
|
||||
p.get("kind"), p["title"], bodyfile,
|
||||
aliases=p.get("aliases") or [], sources=p.get("sources") or [],
|
||||
tags=p.get("tags") or [],
|
||||
date=p.get("date"), domain=p.get("domain", "business"))
|
||||
if rc == 76:
|
||||
gated += 1
|
||||
continue
|
||||
if rc != 0:
|
||||
continue
|
||||
applied += 1
|
||||
original = (p.get("line") or p["title"]).strip()
|
||||
chorus.cmd_append(f"{LOG_DIR}/{today_s}.md",
|
||||
f"- {original} → {p.get('_path', '?')}")
|
||||
print(f"triage: routed {applied}/{len(valid)} item(s); audit in {LOG_DIR}/{today_s}.md. "
|
||||
"Originals kept in the inbox (deletion is explicit-only)."
|
||||
+ (f" {gated} stopped at the duplicate gate — re-propose with the existing "
|
||||
f"entity's title or use capture --merge-into." if gated else ""))
|
||||
return 0
|
||||
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env python3
|
||||
"""migrate.py — bring an existing CHORUS vault up to the plugin's current schema.
|
||||
|
||||
Reads the marker's schema_version and applies each intervening migration in order.
|
||||
Migrations are idempotent and additive; every destructive step (DELETE/move) is
|
||||
gated behind --apply AND printed first. Default mode is a DRY-RUN plan.
|
||||
Cross-platform: pure Python via chorus.py.
|
||||
|
||||
Usage:
|
||||
migrate.py # print the migration plan (no changes)
|
||||
migrate.py --apply # perform the migration (moves/deletes included)
|
||||
|
||||
Env: CHORUS_BASE, CHORUS_KEY (via chorus.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import chorus # noqa: E402
|
||||
|
||||
CURRENT_SCHEMA = 4
|
||||
|
||||
|
||||
def get_text(path: str) -> str | None:
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
chorus.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def list_files(path: str) -> list[str]:
|
||||
if not path.endswith("/"):
|
||||
path += "/"
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
return []
|
||||
chorus.check(status, body, f"ls {path}")
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [entry for entry in payload.get("files", []) if not entry.endswith("/")]
|
||||
|
||||
|
||||
def put_text(path: str, text: str) -> None:
|
||||
status, body = chorus.request("PUT", chorus.vault_url(path), data=text.encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
chorus.check(status, body, f"put {path}")
|
||||
|
||||
|
||||
def delete(path: str) -> None:
|
||||
status, body = chorus.request("DELETE", chorus.vault_url(path))
|
||||
if status != 404:
|
||||
chorus.check(status, body, f"delete {path}")
|
||||
|
||||
|
||||
def move(src: str, dst: str) -> None:
|
||||
text = get_text(src)
|
||||
if text is None:
|
||||
return
|
||||
put_text(dst, text)
|
||||
delete(src)
|
||||
|
||||
|
||||
def do_or_show(apply: bool, desc: str, func=None) -> None:
|
||||
if apply:
|
||||
print(f"migrate: APPLY {desc}")
|
||||
if func:
|
||||
func()
|
||||
else:
|
||||
print(f"migrate: PLAN {desc}")
|
||||
|
||||
|
||||
def find_marker() -> str:
|
||||
"""The vault's marker path: the CHORUS name, else a legacy pre-fork ECHO
|
||||
marker (the rename itself is a future migration step)."""
|
||||
for p in ("_agent/chorus-vault.md", "_agent/echo-vault.md"):
|
||||
if get_text(p) is not None:
|
||||
return p
|
||||
return "_agent/chorus-vault.md"
|
||||
|
||||
|
||||
def marker_schema() -> int:
|
||||
marker = get_text(find_marker())
|
||||
if marker is None:
|
||||
print("migrate: marker missing — vault not bootstrapped. Run bootstrap.py, not migrate.py.")
|
||||
raise SystemExit(3)
|
||||
for line in marker.splitlines():
|
||||
if line.startswith("schema_version:"):
|
||||
try:
|
||||
return int(line.split(":", 1)[1].strip())
|
||||
except ValueError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Migrate CHORUS vault schema")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
start = marker_schema()
|
||||
print(f"migrate: vault schema_version={start}, plugin schema={CURRENT_SCHEMA} "
|
||||
f"{'(APPLY)' if args.apply else '(dry-run)'}")
|
||||
if start >= CURRENT_SCHEMA:
|
||||
print("migrate: up to date — nothing to do.")
|
||||
return 0
|
||||
|
||||
if start < 1:
|
||||
print("migrate: [0->1] retire in-vault control docs (CLAUDE/BOOTSTRAP/STRUCTURE/index.md)")
|
||||
for filename in ["CLAUDE.md", "BOOTSTRAP.md", "STRUCTURE.md", "index.md"]:
|
||||
if get_text(filename) is not None:
|
||||
do_or_show(args.apply, f"delete vault/{filename} (back it up outside the vault first)",
|
||||
lambda f=filename: delete(f))
|
||||
print("migrate: [0->1] reminder: scrub dangling control-doc [[links]] from Related sections.")
|
||||
|
||||
if start < 2:
|
||||
print("migrate: [1->2] fold reviews/ into journal/ and _agent/health/")
|
||||
for filename in list_files("reviews/weekly"):
|
||||
if filename.endswith(".md"):
|
||||
dst = f"journal/weekly/{filename.replace('-review.md', '.md')}"
|
||||
do_or_show(args.apply, f"move reviews/weekly/{filename} -> {dst}",
|
||||
lambda f=filename, d=dst: move(f"reviews/weekly/{f}", d))
|
||||
for filename in list_files("reviews/monthly"):
|
||||
if filename.endswith(".md"):
|
||||
dst = f"_agent/health/{filename}" if filename.endswith("vault-health.md") else f"journal/monthly/{filename}"
|
||||
do_or_show(args.apply, f"move reviews/monthly/{filename} -> {dst}",
|
||||
lambda f=filename, d=dst: move(f"reviews/monthly/{f}", d))
|
||||
for period in ["quarterly", "annual"]:
|
||||
for filename in list_files(f"reviews/{period}"):
|
||||
if filename.endswith(".md"):
|
||||
dst = f"journal/{period}/{filename}"
|
||||
do_or_show(args.apply, f"move reviews/{period}/{filename} -> {dst}",
|
||||
lambda p=period, f=filename, d=dst: move(f"reviews/{p}/{f}", d))
|
||||
print("migrate: [1->2] reminder: update inbound [[reviews/...]] wikilinks manually.")
|
||||
|
||||
if start < 3:
|
||||
print("migrate: [2->3] add the entity-index folder (cross-link / recall feature)")
|
||||
if get_text("_agent/index/README.md") is None:
|
||||
do_or_show(args.apply, "create _agent/index/README.md",
|
||||
lambda: put_text("_agent/index/README.md",
|
||||
"# index\n\nMachine-maintained entity index. See the chorus-memory plugin.\n"))
|
||||
print("migrate: [2->3] then run `sweep.py --apply` to build entities.json and symmetrize links.")
|
||||
|
||||
if start < 4:
|
||||
print("migrate: [3->4] add the recall (BM25) index — hybrid lexical+graph recall")
|
||||
print("migrate: [3->4] _agent/index/ already exists (schema 3); no moves needed.")
|
||||
print("migrate: [3->4] then run `sweep.py --apply` to build _agent/index/recall-index.json.")
|
||||
|
||||
marker_path = find_marker()
|
||||
do_or_show(args.apply, f"set {marker_path} schema_version -> {CURRENT_SCHEMA}",
|
||||
lambda: chorus.cmd_fm(marker_path, "schema_version", str(CURRENT_SCHEMA)))
|
||||
if args.apply:
|
||||
print(f"migrate: migration complete -> schema {CURRENT_SCHEMA}. Run sweep.py --apply, then vault_lint.py.")
|
||||
else:
|
||||
print("migrate: dry-run only. Re-run with --apply to perform the moves/deletes above.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"$comment": "CANONICAL machine-readable routing manifest for the CHORUS vault. This is the single source of truth for 'what paths may be written to'. The human-readable tables in SKILL.md, references/routing-map.md, and references/api-reference.md are DERIVED views of this file — when they disagree, this file wins. vault_lint.py consumes it to enforce the core rule (if a vault path matches no route here, and is not retired, nothing should be written to it); check_routing.py consumes it to verify the human routing docs stay in sync with this manifest. Patterns are Python regexes matched against vault-root-relative paths (no leading slash, no /vault/ prefix).",
|
||||
"schema_version": 2,
|
||||
"routes": [
|
||||
{ "id": "inbox-captures", "pattern": "^inbox/captures/inbox\\.md$", "method": "POST", "trigger": "Destination unknown at capture time", "distinct_because": "Only path whose contract is deferred routing" },
|
||||
{ "id": "inbox-imports", "pattern": "^inbox/imports/[^/]+\\.md$", "method": "PUT", "trigger": "Raw external material dropped wholesale", "distinct_because": "Bulk un-triaged material vs single-line captures" },
|
||||
{ "id": "inbox-processing-log", "pattern": "^inbox/processing-log/\\d{4}-\\d{2}-\\d{2}\\.md$", "method": "POST", "trigger": "An inbox item is routed to its real home", "distinct_because": "Audit trail of moves, not memory itself" },
|
||||
|
||||
{ "id": "journal-daily", "pattern": "^journal/daily/\\d{4}-\\d{2}-\\d{2}\\.md$", "method": "PATCH", "trigger": "First agent activity on a given day", "distinct_because": "Finest grain; PATCHed repeatedly within its period" },
|
||||
{ "id": "journal-weekly", "pattern": "^journal/weekly/\\d{4}-W\\d{2}\\.md$", "method": "PUT", "trigger": "First substantive session of a new ISO week (opt-in)", "distinct_because": "ISO-week grain" },
|
||||
{ "id": "journal-monthly", "pattern": "^journal/monthly/\\d{4}-\\d{2}\\.md$", "method": "PUT", "trigger": "First substantive session of a new month", "distinct_because": "Month grain" },
|
||||
{ "id": "journal-quarterly", "pattern": "^journal/quarterly/\\d{4}-Q[1-4]\\.md$", "method": "PUT", "trigger": "Manual / on request only", "distinct_because": "Strategic grain; never auto-fires" },
|
||||
{ "id": "journal-annual", "pattern": "^journal/annual/\\d{4}\\.md$", "method": "PUT", "trigger": "Manual / on request only", "distinct_because": "Coarsest grain; never auto-fires" },
|
||||
{ "id": "journal-templates", "pattern": "^journal/templates/.+\\.md$", "method": "PUT", "trigger": "Bootstrap seed only", "distinct_because": "Holds templates, not journal content" },
|
||||
|
||||
{ "id": "projects-active", "pattern": "^projects/active/[^/]+\\.md$", "method": "PUT", "trigger": "Work in motion now", "distinct_because": "Default state for anything being worked", "status": "active" },
|
||||
{ "id": "projects-incubating", "pattern": "^projects/incubating/[^/]+\\.md$", "method": "PUT", "trigger": "Idea captured, work not started", "distinct_because": "Pre-work", "status": "incubating" },
|
||||
{ "id": "projects-on-hold", "pattern": "^projects/on-hold/[^/]+\\.md$", "method": "PUT", "trigger": "Paused but still tracked", "distinct_because": "Resumable; not terminal", "status": "on-hold" },
|
||||
{ "id": "projects-archived", "pattern": "^projects/archived/[^/]+\\.md$", "method": "PUT", "trigger": "Done, abandoned, or rolled up", "distinct_because": "Terminal; kept for history", "status": "archived" },
|
||||
{ "id": "projects-template", "pattern": "^projects/project-template\\.md$", "method": "PUT", "trigger": "Bootstrap seed only", "distinct_because": "Template, not a project" },
|
||||
|
||||
{ "id": "areas", "pattern": "^areas/(business|personal|learning|systems)/[^/]+\\.md$", "method": "PUT", "trigger": "Ongoing domain with no finish line", "distinct_because": "No end state — disqualifies it from projects/" },
|
||||
|
||||
{ "id": "resources-people", "pattern": "^resources/people/[^/]+\\.md$", "method": "PUT", "trigger": "A fact about a specific person", "distinct_because": "Keyed to a person" },
|
||||
{ "id": "resources-companies", "pattern": "^resources/companies/[^/]+\\.md$", "method": "PUT", "trigger": "A fact about an organization", "distinct_because": "Keyed to an organization, not an individual" },
|
||||
{ "id": "resources-concepts", "pattern": "^resources/concepts/[^/]+\\.md$", "method": "PUT", "trigger": "A reusable concept/idea", "distinct_because": "An idea vs an external source" },
|
||||
{ "id": "resources-references", "pattern": "^resources/references/[^/]+\\.md$", "method": "PUT", "trigger": "An external source/link worth keeping", "distinct_because": "Points outward" },
|
||||
{ "id": "resources-meetings", "pattern": "^resources/meetings/\\d{4}-\\d{2}-\\d{2}-[^/]+\\.md$", "method": "PUT", "trigger": "Notes tied to a specific meeting", "distinct_because": "Event-anchored to a meeting" },
|
||||
|
||||
{ "id": "decisions-by-date", "pattern": "^decisions/by-date/\\d{4}-\\d{2}-\\d{2}-[^/]+\\.md$", "method": "PUT", "trigger": "A non-obvious decision worth recording", "distinct_because": "Chronological system of record for decisions" },
|
||||
{ "id": "decisions-template", "pattern": "^decisions/decision-template\\.md$", "method": "PUT", "trigger": "Bootstrap seed only", "distinct_because": "Template, not a decision" },
|
||||
|
||||
{ "id": "agent-marker", "pattern": "^_agent/chorus-vault\\.md$", "method": "PUT", "trigger": "Bootstrap / schema migration only", "distinct_because": "Plugin-owned probe; never hand-edited" },
|
||||
{ "id": "agent-marker-legacy", "pattern": "^_agent/echo-vault\\.md$", "method": "GET", "trigger": "Pre-fork ECHO vault adoption (read-only probe)", "distinct_because": "Legacy ECHO marker honored until the rename migration" },
|
||||
{ "id": "agent-context", "pattern": "^_agent/context/[^/]+\\.md$", "method": "PATCH", "trigger": "Active scope changes / task bundles", "distinct_because": "Single live scope pointer + bundles" },
|
||||
{ "id": "agent-semantic", "pattern": "^_agent/memory/semantic/[^/]+\\.md$", "method": "PUT", "trigger": "A durable fact/pattern (incl. operator-preferences.md)", "distinct_because": "Timeless fact" },
|
||||
{ "id": "agent-episodic", "pattern": "^_agent/memory/episodic/[^/]+\\.md$", "method": "PUT", "trigger": "A record of what happened, when", "distinct_because": "Anchored to an event in time" },
|
||||
{ "id": "agent-working", "pattern": "^_agent/memory/working/[^/]+\\.md$", "method": "PUT", "trigger": "Short-lived state for the current effort", "distinct_because": "Explicitly transient" },
|
||||
{ "id": "agent-sessions", "pattern": "^_agent/sessions/\\d{4}-\\d{2}-\\d{2}(-\\d{4})?-[^/]+\\.md$", "method": "PUT", "trigger": "A substantive session ends", "distinct_because": "Per-session record (new ones require HHMM)" },
|
||||
{ "id": "agent-health", "pattern": "^_agent/health/\\d{4}-\\d{2}-vault-health\\.md$", "method": "PUT", "trigger": "First substantive session of a month", "distinct_because": "Vault integrity, not work narrative" },
|
||||
{ "id": "agent-heartbeat", "pattern": "^_agent/heartbeat/[^/]+\\.md$", "method": "PUT", "trigger": "End of every session", "distinct_because": "O(1) orientation pointer; overwritten, never grows" },
|
||||
{ "id": "agent-templates", "pattern": "^_agent/templates/.+\\.md$", "method": "PUT", "trigger": "Bootstrap seed only", "distinct_because": "Holds templates, not memory" },
|
||||
{ "id": "agent-skills-active", "pattern": "^_agent/skills/active/[^/]+\\.md$", "method": "PUT", "trigger": "A skill/plugin catalogued as a capability", "distinct_because": "Catalogs a capability vs the build effort" },
|
||||
{ "id": "agent-skills-archived","pattern": "^_agent/skills/archived/[^/]+\\.md$", "method": "PUT", "trigger": "A catalogued skill is retired", "distinct_because": "Terminal state of the skill catalog" },
|
||||
{ "id": "agent-locks", "pattern": "^_agent/locks/[^/]+\\.lock$", "method": "PUT", "trigger": "Advisory multi-writer lock acquire/release", "distinct_because": "Concurrency coordination, not memory" },
|
||||
{ "id": "agent-index", "pattern": "^_agent/index/[^/]+\\.json$", "method": "PUT", "trigger": "Entity index rebuilt on write/sweep", "distinct_because": "Machine-maintained routing/recall registry, not a note" },
|
||||
|
||||
{ "id": "leaf-readme", "pattern": "^(.+/)?README\\.md$", "method": "PUT", "trigger": "Bootstrap leaf signpost / vault root README", "distinct_because": "Human signpost, not read for routing" }
|
||||
],
|
||||
"retired": [
|
||||
{ "pattern": "^reviews/", "retired_in_schema": 2, "replacement": "journal/{weekly,monthly,quarterly,annual}/ and _agent/health/" },
|
||||
{ "pattern": "^decisions/by-project/", "retired_in_schema": 1, "replacement": "[[wikilink]] under the project's ## Key Decisions" },
|
||||
{ "pattern": "^archive/", "retired_in_schema": 0, "replacement": "projects/archived/ and _agent/skills/archived/" },
|
||||
{ "pattern": "^(CLAUDE|BOOTSTRAP|STRUCTURE|index)\\.md$", "retired_in_schema": 1, "replacement": "All control logic lives in the plugin references/, not the vault" }
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sweep.py — bring an existing CHORUS vault up to the current feature spec.
|
||||
|
||||
After upgrading the plugin (entity index, cross-linking, recall), run this once to
|
||||
backfill what older vaults don't have yet:
|
||||
|
||||
1. ensure the `_agent/index/` folder exists,
|
||||
2. **(re)build the entity index** (`_agent/index/entities.json`) from the notes
|
||||
already in the vault (people, companies, projects, concepts, references,
|
||||
meetings, areas, decisions, semantic/episodic memory, skills),
|
||||
3. **backfill incomplete entity frontmatter** — fill a missing/empty `status:` with
|
||||
the kind default and seed a missing/empty `tags:` with the note's kind, per the
|
||||
KIND_REQUIRED_FM/KIND_STATUS maps in chorus_index (what `/chorus-health` flags as
|
||||
`incomplete-frontmatter`),
|
||||
4. **symmetrize cross-links** — for every `## Related` link A -> B, ensure the
|
||||
reciprocal B -> A exists (it only adds the missing direction; it never invents
|
||||
new links from body text), and
|
||||
5. stamp the marker `schema_version` to the current schema (4).
|
||||
|
||||
Dry-run by default; pass --apply to write. READ-ONLY without --apply.
|
||||
Cross-platform: pure Python via chorus.py.
|
||||
|
||||
Usage: sweep.py [--apply]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import chorus # noqa: E402
|
||||
import chorus_index as idx_mod # noqa: E402
|
||||
import chorus_links as links # noqa: E402
|
||||
import chorus_recall # noqa: E402
|
||||
|
||||
CURRENT_SCHEMA = 4
|
||||
TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
SKIP_BASENAMES = {"README.md"}
|
||||
|
||||
|
||||
kind_for = idx_mod.kind_for_path
|
||||
|
||||
|
||||
def fm_block(text: str) -> str:
|
||||
if not text.startswith("---"):
|
||||
return ""
|
||||
end = text.find("\n---", 3)
|
||||
return text[:end] if end != -1 else text
|
||||
|
||||
|
||||
def fm_populated(text: str, field: str) -> bool:
|
||||
"""True when the frontmatter carries a real value for `field` (flow list `[x]`,
|
||||
block list items, or a non-empty scalar). Mirrors vault_lint.fm_field_populated."""
|
||||
fm = fm_block(text)
|
||||
m = re.search(rf"(?m)^{re.escape(field)}:[ \t]*(.*)$", fm)
|
||||
if not m:
|
||||
return False
|
||||
val = m.group(1).strip()
|
||||
if val == "[]":
|
||||
return False
|
||||
if val:
|
||||
return True
|
||||
rest = fm[m.end():].lstrip("\n")
|
||||
first = rest.splitlines()[0] if rest else ""
|
||||
return bool(re.match(r"^\s+-\s*\S", first))
|
||||
|
||||
|
||||
def get(path: str) -> str | None:
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
chorus.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def list_dir(path: str):
|
||||
p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/")
|
||||
body = get(p)
|
||||
if body is None:
|
||||
return [], []
|
||||
try:
|
||||
j = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [], []
|
||||
entries = list(j.get("files", [])) + list(j.get("folders", []))
|
||||
files = [e for e in entries if not e.endswith("/")]
|
||||
folders = [e[:-1] for e in entries if e.endswith("/")]
|
||||
return files, folders
|
||||
|
||||
|
||||
def walk(prefix: str = ""):
|
||||
files, folders = list_dir(prefix)
|
||||
for f in files:
|
||||
yield prefix + f
|
||||
for d in folders:
|
||||
yield from walk(f"{prefix}{d}/")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Bring an CHORUS vault up to the current feature spec")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args(argv)
|
||||
apply = args.apply
|
||||
tag = "APPLY" if apply else "PLAN "
|
||||
|
||||
if get("_agent/chorus-vault.md") is None and get("_agent/echo-vault.md") is None:
|
||||
print("sweep: marker missing — vault not bootstrapped. Run bootstrap.py first.", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
print(f"sweep: {'applying changes' if apply else 'dry-run (no writes)'}\n")
|
||||
|
||||
# ---- 1. index folder ------------------------------------------------------
|
||||
if get("_agent/index/README.md") is None:
|
||||
print(f"sweep: {tag} create _agent/index/README.md")
|
||||
if apply:
|
||||
chorus.request("PUT", chorus.vault_url("_agent/index/README.md"),
|
||||
data=b"# index\n\nMachine-maintained entity index. See the chorus-memory plugin.\n",
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
|
||||
all_files = [p for p in walk()
|
||||
if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
|
||||
and not TEMPLATE_RE.search(p)]
|
||||
|
||||
# Fetch every note's content ONCE, concurrently. All three passes below (index,
|
||||
# recall, link symmetrize) read from this shared cache, so no file is fetched
|
||||
# twice and link targets aren't re-fetched per reference.
|
||||
texts = chorus.read_many(all_files)
|
||||
print(f"sweep: read {len(texts)} notes (concurrent, keep-alive)\n")
|
||||
|
||||
# ---- 2. (re)build entity index -------------------------------------------
|
||||
index = idx_mod.load()
|
||||
existing = dict(index.get("entities", {}))
|
||||
rebuilt = idx_mod.empty_index()
|
||||
added = updated = 0
|
||||
for path in all_files:
|
||||
kind = kind_for(path)
|
||||
if kind is None:
|
||||
continue
|
||||
base = path.rsplit("/", 1)[-1][:-3]
|
||||
slug = idx_mod.slugify(base)
|
||||
text = texts.get(path) or ""
|
||||
title = links.first_h1(text) or base
|
||||
prev = existing.get(slug)
|
||||
# Frontmatter aliases are authoritative and re-folded every rebuild; prior index
|
||||
# aliases (e.g. mentions learned by capture) are preserved; upsert adds title variants.
|
||||
prev_aliases = prev.get("aliases", []) if prev else []
|
||||
aliases = list(prev_aliases) + idx_mod.aliases_in_frontmatter(text)
|
||||
idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
|
||||
if not prev:
|
||||
added += 1
|
||||
elif prev.get("path") != path or prev.get("title") != title:
|
||||
updated += 1
|
||||
print(f"sweep: {tag} entity index — {len(rebuilt['entities'])} entities "
|
||||
f"({added} new, {updated} changed)")
|
||||
if apply:
|
||||
idx_mod.save(rebuilt)
|
||||
|
||||
# ---- 2b. (re)build the recall (BM25) index -------------------------------
|
||||
if apply:
|
||||
rix = chorus_recall.rebuild(prefetched=texts)
|
||||
print(f"sweep: {tag} recall index — {rix.n_docs} docs (BM25)")
|
||||
else:
|
||||
n_idx = sum(1 for p in all_files if chorus_recall._indexable(p))
|
||||
print(f"sweep: {tag} recall index — would rebuild BM25 over {n_idx} corpus notes "
|
||||
f"(entities + sessions/journal)")
|
||||
|
||||
# ---- 3. backfill incomplete entity frontmatter ----------------------------
|
||||
# What /chorus-health flags as incomplete-frontmatter, filled mechanically: a missing
|
||||
# or empty status gets the kind default; missing/empty tags get the kind as a
|
||||
# baseline tag. cmd_fm is create-or-replace, so absent keys are inserted surgically.
|
||||
fixes: list[tuple[str, str, str]] = []
|
||||
for path in all_files:
|
||||
kind = kind_for(path)
|
||||
if kind is None:
|
||||
continue
|
||||
text = texts.get(path) or ""
|
||||
if not text.startswith("---"):
|
||||
continue # no frontmatter block at all — lint flags it; not a mechanical fill
|
||||
for field in idx_mod.KIND_REQUIRED_FM.get(kind, ()):
|
||||
if not fm_populated(text, field):
|
||||
value = (idx_mod.KIND_STATUS.get(kind, "active") if field == "status"
|
||||
else json.dumps([kind]))
|
||||
fixes.append((path, field, value))
|
||||
print(f"sweep: {tag} frontmatter backfill — {len(fixes)} field(s) to fill")
|
||||
for path, field, value in fixes[:40]:
|
||||
print(f" {path}: {field} -> {value}")
|
||||
if len(fixes) > 40:
|
||||
print(f" ... and {len(fixes) - 40} more")
|
||||
if apply:
|
||||
for path, field, value in fixes:
|
||||
chorus.cmd_fm(path, field, value)
|
||||
|
||||
# ---- 4. symmetrize existing Related links --------------------------------
|
||||
fileset = set(all_files)
|
||||
basemap: dict[str, str] = {}
|
||||
for p in all_files:
|
||||
basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)
|
||||
|
||||
def resolve_target(t: str) -> str | None:
|
||||
t = t.strip()
|
||||
if "/" in t:
|
||||
cand = t if t.endswith(".md") else t + ".md"
|
||||
return cand if cand in fileset else None
|
||||
return basemap.get(idx_mod.slugify(t))
|
||||
|
||||
missing = set() # (target_path, source_path) — reciprocal link to add on target
|
||||
for path in all_files:
|
||||
text = texts.get(path)
|
||||
if text is None:
|
||||
continue
|
||||
for tgt in links.related_targets(text):
|
||||
tp = resolve_target(tgt)
|
||||
if not tp or tp == path:
|
||||
continue
|
||||
back = {resolve_target(t) for t in links.related_targets(texts.get(tp) or "")}
|
||||
if path not in back:
|
||||
missing.add((tp, path))
|
||||
missing = sorted(missing)
|
||||
print(f"sweep: {tag} reciprocal links — {len(missing)} to add")
|
||||
for tp, sp in missing[:40]:
|
||||
print(f" {tp} += [[{links.link_token(sp)}]]")
|
||||
if len(missing) > 40:
|
||||
print(f" ... and {len(missing) - 40} more")
|
||||
if apply:
|
||||
for tp, sp in missing:
|
||||
links.add_one(tp, sp)
|
||||
|
||||
# ---- 5. stamp schema ------------------------------------------------------
|
||||
marker_path = "_agent/chorus-vault.md"
|
||||
marker = get(marker_path)
|
||||
if marker is None: # adopted pre-fork ECHO vault — stamp the marker it has
|
||||
marker_path = "_agent/echo-vault.md"
|
||||
marker = get(marker_path) or ""
|
||||
cur = next((int(m) for m in re.findall(r"(?m)^schema_version:\s*(\d+)", marker)), 0)
|
||||
if cur < CURRENT_SCHEMA:
|
||||
print(f"sweep: {tag} schema_version {cur} -> {CURRENT_SCHEMA}")
|
||||
if apply:
|
||||
chorus.cmd_fm(marker_path, "schema_version", str(CURRENT_SCHEMA))
|
||||
|
||||
print(f"\nsweep: {'done — run vault_lint.py to confirm invariants' if apply else 'dry-run complete. Re-run with --apply to write.'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,233 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline regression tests for the CHORUS tooling. No network, no vault.
|
||||
|
||||
Run: python3 test_chorus_client.py (or: pytest test_chorus_client.py)
|
||||
|
||||
Covers chorus.py body/scalar normalization, the routing-view consistency checker's
|
||||
helpers, and — as the guard for improvement #4 — asserts the shipped docs are in
|
||||
sync with routing.json.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import chorus # noqa: E402
|
||||
import check_routing # noqa: E402
|
||||
import chorus_index # noqa: E402
|
||||
import chorus_links # noqa: E402
|
||||
|
||||
|
||||
def test_heading_replace_body_is_newline_guarded() -> None:
|
||||
assert chorus.normalize_patch_body(b"- [x] done\n", "replace", "heading") == b"\n- [x] done\n"
|
||||
|
||||
|
||||
def test_heading_append_body_ends_with_newline() -> None:
|
||||
assert chorus.normalize_patch_body(b"- item", "append", "heading") == b"- item\n"
|
||||
|
||||
|
||||
def test_frontmatter_plain_string_becomes_json_scalar() -> None:
|
||||
assert json.loads(chorus.normalize_json_scalar("Fabrication Lead")) == "Fabrication Lead"
|
||||
|
||||
|
||||
def test_frontmatter_existing_json_is_preserved() -> None:
|
||||
assert chorus.normalize_json_scalar('"2026-06-20"') == b'"2026-06-20"'
|
||||
|
||||
|
||||
def test_read_many_dedups_and_maps_each_path() -> None:
|
||||
# Concurrency helper contract, no network: monkeypatch the per-file fetch.
|
||||
orig = chorus.get_text
|
||||
chorus.get_text = lambda p: f"T:{p}"
|
||||
try:
|
||||
assert chorus.read_many(["a", "b", "a"]) == {"a": "T:a", "b": "T:b"}
|
||||
finally:
|
||||
chorus.get_text = orig
|
||||
|
||||
|
||||
def test_read_many_empty_returns_empty_dict() -> None:
|
||||
assert chorus.read_many([]) == {}
|
||||
|
||||
|
||||
def test_read_many_tolerates_unreadable_file_as_none() -> None:
|
||||
orig = chorus.get_text
|
||||
chorus.get_text = lambda p: None if p == "bad" else f"T:{p}"
|
||||
try:
|
||||
assert chorus.read_many(["ok", "bad"]) == {"ok": "T:ok", "bad": None}
|
||||
finally:
|
||||
chorus.get_text = orig
|
||||
|
||||
|
||||
def test_plugin_description_under_500_chars() -> None:
|
||||
# Marketplace cap — the description has silently regressed past it before (a 525-char
|
||||
# relapse + an earlier "fix description length" commit). build.py fails the build on this;
|
||||
# this guards it in CI so it's caught before packaging.
|
||||
manifest = Path(__file__).resolve().parents[3] / ".claude-plugin" / "plugin.json"
|
||||
desc = json.loads(manifest.read_text(encoding="utf-8")).get("description", "")
|
||||
assert 0 < len(desc) < 500, f"plugin description is {len(desc)} chars (must be under 500)"
|
||||
|
||||
|
||||
def test_stem_extracts_literal_directory_prefix() -> None:
|
||||
assert check_routing.stem(r"^projects/active/[^/]+\.md$") == "projects/active/"
|
||||
assert check_routing.stem(r"^areas/(business|personal)/[^/]+\.md$") == "areas/"
|
||||
assert check_routing.stem(r"^journal/daily/\d{4}-\d{2}-\d{2}\.md$") == "journal/daily/"
|
||||
|
||||
|
||||
def test_concretize_substitutes_placeholders() -> None:
|
||||
assert check_routing.concretize("projects/<lifecycle>/<slug>.md") == "projects/active/sample.md"
|
||||
assert check_routing.concretize("_agent/sessions/YYYY-MM-DD-HHMM-<slug>.md") == "_agent/sessions/2026-01-02-0900-sample.md"
|
||||
assert check_routing.concretize("journal/weekly/YYYY-Www.md") == "journal/weekly/2026-W01.md"
|
||||
|
||||
|
||||
def test_looks_like_path_filters_non_paths() -> None:
|
||||
assert check_routing.looks_like_path("inbox/captures/inbox.md")
|
||||
assert check_routing.looks_like_path("_agent/locks/vault.lock")
|
||||
assert not check_routing.looks_like_path("application/vnd.olrapi.document-map+json")
|
||||
assert not check_routing.looks_like_path("Operator Preferences::Fact / Pattern")
|
||||
assert not check_routing.looks_like_path("status: active")
|
||||
|
||||
|
||||
def test_docs_are_in_sync_with_routing_json() -> None:
|
||||
# The guard for improvement #4: SKILL.md / routing-map.md / api-reference.md
|
||||
# must stay consistent with routing.json.
|
||||
assert check_routing.main() == 0
|
||||
|
||||
|
||||
def test_index_slugify_and_derive_path() -> None:
|
||||
assert chorus_index.slugify("Bob Smith!") == "bob-smith"
|
||||
assert chorus_index.derive_path("person", "bob-smith") == "resources/people/bob-smith.md"
|
||||
assert chorus_index.derive_path("project", "chorus") == "projects/active/chorus.md"
|
||||
assert chorus_index.derive_path("area", "ops", domain="business") == "areas/business/ops.md"
|
||||
assert chorus_index.derive_path("decision", "x", date="2026-01-02") == "decisions/by-date/2026-01-02-x.md"
|
||||
|
||||
|
||||
def test_index_resolve_matches_alias_and_title() -> None:
|
||||
index = {"entities": {"bob-smith": {"path": "resources/people/bob-smith.md",
|
||||
"kind": "person", "title": "Bob Smith", "aliases": ["bob"]}}}
|
||||
assert chorus_index.resolve(index, "Bob Smith")[0] == "bob-smith"
|
||||
assert chorus_index.resolve(index, "bob")[0] == "bob-smith"
|
||||
assert chorus_index.resolve(index, "nobody")[0] is None
|
||||
|
||||
|
||||
def test_kind_for_path() -> None:
|
||||
assert chorus_index.kind_for_path("resources/people/x.md") == "person"
|
||||
assert chorus_index.kind_for_path("projects/on-hold/x.md") == "project"
|
||||
assert chorus_index.kind_for_path("journal/daily/2026-01-01.md") is None
|
||||
|
||||
|
||||
def test_derive_aliases_produces_punctuation_variants() -> None:
|
||||
al = chorus_index.derive_aliases("CHORUS Memory")
|
||||
assert "chorus-memory" in al and "chorus memory" in al
|
||||
assert chorus_index.derive_aliases("") == []
|
||||
|
||||
|
||||
def test_upsert_folds_in_title_variants_and_drops_slug() -> None:
|
||||
index = {"entities": {}}
|
||||
e = chorus_index.upsert(index, "chorus-memory", "projects/active/chorus-memory.md",
|
||||
"project", "Chorus Memory")
|
||||
assert "chorus memory" in e["aliases"] # space variant auto-derived
|
||||
assert "chorus-memory" not in e["aliases"] # equals the slug -> not stored redundantly
|
||||
|
||||
|
||||
def test_fuzzy_candidates_finds_shortened_name() -> None:
|
||||
# The real bug: a project slugged "chorus" must surface for the mention "chorus memory".
|
||||
index = {"entities": {"chorus": {"path": "projects/active/chorus.md", "kind": "project",
|
||||
"title": "chorus", "aliases": []}}}
|
||||
cands = chorus_index.fuzzy_candidates(index, "chorus memory")
|
||||
assert cands and cands[0][0] == "chorus"
|
||||
# exact resolve still misses (it's exact-only) — fuzzy is the safety net:
|
||||
assert chorus_index.resolve(index, "chorus memory")[0] is None
|
||||
|
||||
|
||||
def test_fuzzy_candidates_ignores_common_and_short_tokens() -> None:
|
||||
index = {"entities": {"data": {"path": "x.md", "kind": "concept",
|
||||
"title": "data", "aliases": []}}}
|
||||
assert chorus_index.fuzzy_candidates(index, "data pipeline") == []
|
||||
|
||||
|
||||
def test_gate_candidates_blocks_same_kind_rare_token() -> None:
|
||||
# The blocking case the gate exists for: same kind, and the shared token is unique
|
||||
# to that one entity — "Robert Smith" vs the person bob-smith must gate.
|
||||
index = {"entities": {"bob-smith": {"path": "resources/people/bob-smith.md",
|
||||
"kind": "person", "title": "Bob Smith",
|
||||
"aliases": []}}}
|
||||
gated = chorus_index.gate_candidates(index, "Robert Smith", kind="person")
|
||||
assert gated and gated[0][0] == "bob-smith"
|
||||
|
||||
|
||||
def test_gate_candidates_skips_cross_kind() -> None:
|
||||
# A company/decision named after a person or project is intentional naming, not a
|
||||
# duplicate — cross-kind lookalikes warn but never block.
|
||||
index = {"entities": {"bob-smith": {"path": "resources/people/bob-smith.md",
|
||||
"kind": "person", "title": "Bob Smith",
|
||||
"aliases": []}}}
|
||||
assert chorus_index.gate_candidates(index, "Smith Consulting", kind="company") == []
|
||||
# ...but the advisory candidate list still surfaces it:
|
||||
assert chorus_index.fuzzy_candidates(index, "Smith Consulting")
|
||||
|
||||
|
||||
def test_gate_candidates_skips_single_vault_common_token() -> None:
|
||||
# The live 1.5.0 false positive: "chorus" appears in many entity names, so any title
|
||||
# containing it scored 1.0 against every single-token "chorus" entity. A lone shared
|
||||
# token only blocks when it is unique to that entity (df == 1).
|
||||
ents = {
|
||||
"chorus": {"path": "projects/active/chorus.md", "kind": "project",
|
||||
"title": "chorus", "aliases": []},
|
||||
"chorus-v-05": {"path": "projects/archived/chorus-v.05.md", "kind": "project",
|
||||
"title": "chorus-v.05", "aliases": []},
|
||||
}
|
||||
assert chorus_index.gate_candidates({"entities": ents}, "chorus handbook",
|
||||
kind="project") == []
|
||||
# multi-token overlap still blocks, even when the individual tokens are common:
|
||||
ents["chorus-memory-system"] = {"path": "projects/active/chorus-memory-system.md",
|
||||
"kind": "project", "title": "chorus memory system",
|
||||
"aliases": []}
|
||||
ents["chorus-memory-notes"] = {"path": "resources/concepts/chorus-memory-notes.md",
|
||||
"kind": "concept", "title": "chorus memory notes",
|
||||
"aliases": []}
|
||||
gated = chorus_index.gate_candidates({"entities": ents}, "chorus memory platform",
|
||||
kind="project")
|
||||
assert [s for s, _, _ in gated] == ["chorus-memory-system"]
|
||||
|
||||
|
||||
def test_aliases_in_frontmatter_flow_and_block() -> None:
|
||||
flow = '---\ntype: project\naliases: ["chorus memory", "chorus plugin"]\n---\n# x\n'
|
||||
assert chorus_index.aliases_in_frontmatter(flow) == ["chorus memory", "chorus plugin"]
|
||||
block = "---\ntype: project\naliases:\n - chorus memory\n - chorus plugin\n---\n# x\n"
|
||||
assert chorus_index.aliases_in_frontmatter(block) == ["chorus memory", "chorus plugin"]
|
||||
assert chorus_index.aliases_in_frontmatter("# no frontmatter\n") == []
|
||||
|
||||
|
||||
def test_links_add_related_is_idempotent_and_bidirectional_token() -> None:
|
||||
text = "---\ntype: person\n---\n\n# Bob\n\n## Related\n"
|
||||
new, changed = chorus_links.add_related(text, "resources/companies/acme.md")
|
||||
assert changed and "[[resources/companies/acme]]" in new
|
||||
again, changed2 = chorus_links.add_related(new, "resources/companies/acme.md")
|
||||
assert not changed2 and again == new
|
||||
|
||||
|
||||
def test_links_create_related_section_when_absent() -> None:
|
||||
text = "---\ntype: concept\n---\n\n# Alpha\n\nbody\n"
|
||||
new, changed = chorus_links.add_related(text, "resources/concepts/beta.md")
|
||||
assert changed and "## Related" in new and "[[resources/concepts/beta]]" in new
|
||||
|
||||
|
||||
def _run_all() -> int:
|
||||
tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
|
||||
failed = 0
|
||||
for test in tests:
|
||||
try:
|
||||
test()
|
||||
print(f"ok {test.__name__}")
|
||||
except AssertionError as exc:
|
||||
failed += 1
|
||||
print(f"FAIL {test.__name__}: {exc}")
|
||||
print(f"\n{len(tests) - failed}/{len(tests)} passed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_run_all())
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""test_v1_scaffold.py — H4 interface guard for the v1.0 scaffolds. [offline, no creds]
|
||||
|
||||
Asserts the scaffold contract holds so the skeleton can't silently rot while 1.0 is
|
||||
built: every planned module imports, the parts that ARE implemented work (BM25 ranking,
|
||||
queue/cache round-trip, slug helpers, secret resolution order), and the parts that are
|
||||
TODO raise NotImplementedError (so a stub can't masquerade as done).
|
||||
|
||||
Run: python test_v1_scaffold.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
failures: list[str] = []
|
||||
|
||||
|
||||
def check(name: str, cond: bool, detail: str = "") -> None:
|
||||
print(f"{'ok ' if cond else 'FAIL'} {name}" + (f" -- {detail}" if not cond else ""))
|
||||
if not cond:
|
||||
failures.append(name)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# H1 — BM25 core is implemented; assert it actually ranks a relevant doc first.
|
||||
import chorus_recall as r
|
||||
ix = r.Bm25Index()
|
||||
ix.add("notes/mqtt.md", "the mqtt broker config and tls certificate rotation")
|
||||
ix.add("notes/payroll.md", "quarterly payroll run and employee deductions")
|
||||
top = ix.score("certificate rotation broker", limit=2)
|
||||
check("H1 BM25 ranks relevant doc first", bool(top) and top[0][0] == "notes/mqtt.md", str(top))
|
||||
check("H1 BM25 survives json round-trip",
|
||||
r.Bm25Index.from_json(ix.to_json()).score("payroll")[0][0] == "notes/payroll.md")
|
||||
ix.remove("notes/payroll.md")
|
||||
check("H1 BM25 remove() drops a doc's postings", ix.score("payroll") == [])
|
||||
check("H1 recall path is wired (not a stub)",
|
||||
all(callable(getattr(r, n, None))
|
||||
for n in ("recall", "rebuild", "update_note", "load_index", "save_index")))
|
||||
|
||||
# H2 — cache round-trips, enqueue dedups, flush is a no-op on an empty queue (offline-safe).
|
||||
import chorus_queue as q
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
os.environ["CHORUS_STATE_DIR"] = d
|
||||
q.cache_put("a/b.md", b"hello")
|
||||
check("H2 cache round-trip", q.cache_get("a/b.md") == b"hello")
|
||||
q.enqueue("PUT", "http://x/vault/a.md", b"body", {}, idem_key="k1")
|
||||
check("H2 enqueue persists a record", len(q.pending()) == 1)
|
||||
q.enqueue("PUT", "http://x/vault/a.md", b"body", {}, idem_key="k1")
|
||||
check("H2 enqueue dedups by idem_key", len(q.pending()) == 1)
|
||||
with tempfile.TemporaryDirectory() as d2:
|
||||
os.environ["CHORUS_STATE_DIR"] = d2
|
||||
check("H2 flush() returns 0 on an empty queue (no network)", q.flush() == 0)
|
||||
|
||||
# H3 — lock CM + atomic index update are stubs.
|
||||
import chorus_concurrency as c
|
||||
check("H3 auto_owner is stable", c.auto_owner() == c.auto_owner())
|
||||
check("H3 vault_lock returns a context manager (no network until entered)",
|
||||
hasattr(c.vault_lock(), "__enter__") and hasattr(c.vault_lock(), "__exit__"))
|
||||
check("H3 atomic_index_update is wired", callable(c.atomic_index_update))
|
||||
|
||||
# H5 — proposal validation is pure (offline): keeps valid (incl. inbox), drops the rest.
|
||||
import chorus_reflect as rf
|
||||
v, errs = rf.validate([
|
||||
{"title": "Ok", "kind": "person", "confidence": 0.9},
|
||||
{"title": "", "kind": "person"}, # missing title
|
||||
{"title": "Bad", "kind": "bogus"}, # unknown kind
|
||||
{"title": "Weak", "kind": "person", "confidence": 0.2}, # below confidence floor
|
||||
{"title": "Note", "inbox": True}, # inbox needs no kind
|
||||
])
|
||||
check("H5 validate keeps valid (incl. inbox), drops invalid", len(v) == 2 and len(errs) == 3, (len(v), errs))
|
||||
check("H5 classify/preview/apply wired",
|
||||
all(callable(getattr(rf, n, None)) for n in ("classify", "preview", "apply")))
|
||||
|
||||
# Config — owner/endpoint/key resolve from env -> machine-local config.json (no baked
|
||||
# defaults). Env wins per field; the file fills the rest; missing required -> raises.
|
||||
import chorus_config as cfg
|
||||
saved_env = {k: os.environ.get(k) for k in ("CHORUS_KEY", "CHORUS_BASE", "CHORUS_OWNER", "CHORUS_CONFIG")}
|
||||
try:
|
||||
for k in saved_env:
|
||||
os.environ.pop(k, None)
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
cfgfile = os.path.join(d, "config.json")
|
||||
os.environ["CHORUS_CONFIG"] = cfgfile
|
||||
# init writes the placeholder template only when absent.
|
||||
p, created = cfg.init_template()
|
||||
check("config init scaffolds the template", created and os.path.exists(cfgfile))
|
||||
p2, created2 = cfg.init_template()
|
||||
check("config init is idempotent (no clobber)", created2 is False)
|
||||
# write_config persists a filled-in config; load() reads it back.
|
||||
cfg.write_config("Owner Name", "https://obsidian.example.com/", "file-key-xyz")
|
||||
loaded = cfg.load()
|
||||
check("config file round-trips owner/endpoint/key",
|
||||
loaded == {"owner": "Owner Name",
|
||||
"endpoint": "https://obsidian.example.com", # trailing / stripped
|
||||
"key": "file-key-xyz"})
|
||||
# env overrides the file, per field.
|
||||
os.environ["CHORUS_KEY"] = "env-key-123"
|
||||
check("env CHORUS_KEY overrides config", cfg.resolve_key() == "env-key-123")
|
||||
os.environ.pop("CHORUS_KEY", None)
|
||||
# missing required field raises a helpful error.
|
||||
cfg.write_config("Owner Name", "", "")
|
||||
try:
|
||||
cfg.resolve_key()
|
||||
check("missing key raises", False)
|
||||
except RuntimeError:
|
||||
check("missing key raises", True)
|
||||
finally:
|
||||
for k, v in saved_env.items():
|
||||
if v is None:
|
||||
os.environ.pop(k, None)
|
||||
else:
|
||||
os.environ[k] = v
|
||||
|
||||
# M2 — slug collision + link-confidence helpers are implemented.
|
||||
import chorus_quality as qual
|
||||
check("M2 safe_slug disambiguates a collision",
|
||||
qual.safe_slug({"foo"}, "foo") != "foo")
|
||||
check("M2 short/common tokens are not confident links",
|
||||
qual.is_confident_link("API", "we built an api") is False)
|
||||
|
||||
# M3 — doctor module exposes run(); M4 — output envelope is implemented.
|
||||
import chorus_doctor as doc
|
||||
check("M3 doctor exposes run()", hasattr(doc, "run"))
|
||||
import chorus_output as out
|
||||
env = out.envelope("created", {"path": "p.md"})
|
||||
check("M4 envelope shape", env.get("ok") is True and env.get("action") == "created")
|
||||
|
||||
print(f"\n{len(failures)} failure(s)" if failures else "\nall scaffold interface checks passed")
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,360 @@
|
||||
#!/usr/bin/env python3
|
||||
"""vault_lint.py — mechanically assert CHORUS vault invariants. READ-ONLY.
|
||||
|
||||
Catches the recurring drift bugs prose rules can't enforce: folder<->status
|
||||
mismatch, duplicate slugs, wikilinks leaking into frontmatter, duplicate
|
||||
"## Agent Log" headings, stale active projects, aging inbox captures, impossible
|
||||
dates, missing frontmatter, broken source_notes, scope drift, and paths that no
|
||||
route in routing.json permits. Cross-platform: pure Python via chorus.py.
|
||||
|
||||
Exit status: 0 = clean, 1 = violations found, 2 = vault unreachable,
|
||||
3 = vault not bootstrapped (marker missing).
|
||||
|
||||
Config (env overrides):
|
||||
CHORUS_BASE, CHORUS_KEY (via chorus.py)
|
||||
CHORUS_TODAY (default machine date) — pass the conversation's currentDate so
|
||||
stale/aging math uses the same clock the agent writes with.
|
||||
STALE_DAYS (default 30) INBOX_DAYS (default 14) SCOPE_STALE_SESSIONS (default 3)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import chorus # noqa: E402
|
||||
import chorus_index as idx_mod # noqa: E402
|
||||
import chorus_links as links # noqa: E402
|
||||
|
||||
TODAY = dt.date.fromisoformat(os.environ.get("CHORUS_TODAY") or dt.date.today().isoformat())
|
||||
STALE_DAYS = int(os.environ.get("STALE_DAYS", "30"))
|
||||
INBOX_DAYS = int(os.environ.get("INBOX_DAYS", "14"))
|
||||
SCOPE_STALE_SESSIONS = int(os.environ.get("SCOPE_STALE_SESSIONS", "3"))
|
||||
LIFECYCLES = ["active", "incubating", "on-hold", "archived"]
|
||||
SKIP = {"README.md", "project-template.md", "decision-template.md"}
|
||||
REQUIRED_FM = ("type", "created")
|
||||
|
||||
violations: list[tuple[str, str]] = []
|
||||
|
||||
|
||||
def flag(check: str, message: str) -> None:
|
||||
violations.append((check, message))
|
||||
|
||||
|
||||
def get(path: str) -> str | None:
|
||||
status, body = chorus.request("GET", chorus.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
chorus.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def list_dir(path: str) -> tuple[list[str], list[str]]:
|
||||
if path and not path.endswith("/"):
|
||||
path += "/"
|
||||
body = get(path)
|
||||
if body is None:
|
||||
return [], []
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [], []
|
||||
entries = list(payload.get("files", [])) + list(payload.get("folders", []))
|
||||
files = [entry for entry in entries if not entry.endswith("/")]
|
||||
folders = [entry[:-1] for entry in entries if entry.endswith("/")]
|
||||
return files, folders
|
||||
|
||||
|
||||
def walk(prefix: str = ""):
|
||||
files, folders = list_dir(prefix)
|
||||
for filename in files:
|
||||
yield prefix + filename
|
||||
for folder in folders:
|
||||
yield from walk(f"{prefix}{folder}/")
|
||||
|
||||
|
||||
def split_frontmatter(text: str) -> tuple[str, str]:
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return "", text
|
||||
for index in range(1, len(lines)):
|
||||
if lines[index].strip() == "---":
|
||||
return "\n".join(lines[1:index]), "\n".join(lines[index + 1:])
|
||||
return "", text
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[str, dict[str, object]]:
|
||||
raw, _ = split_frontmatter(text)
|
||||
fields: dict[str, object] = {}
|
||||
for line in raw.splitlines():
|
||||
match = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||||
if not match:
|
||||
continue
|
||||
value: object = match.group(2).strip().strip('"').strip("'")
|
||||
if isinstance(value, str) and value.startswith("[") and value.endswith("]"):
|
||||
value = [part.strip().strip('"').strip("'") for part in value[1:-1].split(",") if part.strip()]
|
||||
fields[match.group(1)] = value
|
||||
return raw, fields
|
||||
|
||||
|
||||
def parse_date(value: object) -> dt.date | None:
|
||||
match = re.match(r"(\d{4}-\d{2}-\d{2})", str(value or ""))
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return dt.date.fromisoformat(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def as_list(value: object) -> list[object]:
|
||||
if value in (None, ""):
|
||||
return []
|
||||
return value if isinstance(value, list) else [value]
|
||||
|
||||
|
||||
def fm_field_populated(raw: str, fields: dict[str, object], field: str) -> bool:
|
||||
"""True when `field` exists AND holds a real value. parse_frontmatter is line-based,
|
||||
so a block-style list (`tags:\n - x`) parses as "" — check the raw text for items
|
||||
before calling the field empty."""
|
||||
value = fields.get(field)
|
||||
if isinstance(value, list):
|
||||
return bool(value)
|
||||
if str(value or "").strip():
|
||||
return True
|
||||
lines = raw.splitlines()
|
||||
for index, line in enumerate(lines):
|
||||
if re.match(rf"^{re.escape(field)}:\s*$", line):
|
||||
return index + 1 < len(lines) and bool(re.match(r"^\s+-\s*\S", lines[index + 1]))
|
||||
return False
|
||||
|
||||
|
||||
def route_matchers():
|
||||
routing = json.loads((SCRIPT_DIR / "routing.json").read_text(encoding="utf-8"))
|
||||
routes = [(item["id"], re.compile(item["pattern"])) for item in routing.get("routes", [])]
|
||||
retired = [(re.compile(item["pattern"]), item.get("replacement", "")) for item in routing.get("retired", [])]
|
||||
return routes, retired
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
if get("_agent/chorus-vault.md") is None and get("_agent/echo-vault.md") is None:
|
||||
print("vault-lint: marker missing — vault not bootstrapped (run bootstrap.py).", file=sys.stderr)
|
||||
return 3
|
||||
except Exception as exc:
|
||||
print(f"vault-lint: vault unreachable ({exc}).", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
routes, retired = route_matchers()
|
||||
except Exception as exc:
|
||||
routes, retired = [], []
|
||||
flag("routing-manifest", f"could not load routing.json ({exc}) — path checks skipped")
|
||||
|
||||
all_files = list(walk())
|
||||
md_files = [p for p in all_files if p.endswith(".md")]
|
||||
md_set = set(md_files)
|
||||
# Fetch every markdown note ONCE, concurrently; all per-note checks below read from
|
||||
# this shared cache instead of issuing a fresh GET per file per section.
|
||||
texts = chorus.read_many(md_files)
|
||||
|
||||
# Path membership + retired-path detection
|
||||
for path in all_files:
|
||||
if routes and not any(rx.match(path) for _, rx in routes):
|
||||
replacement = next((repl for rx, repl in retired if rx.match(path)), None)
|
||||
if replacement is not None:
|
||||
flag("retired-path", f"{path}: retired location — should be {replacement}")
|
||||
else:
|
||||
flag("unknown-path", f"{path}: matches no route in routing.json")
|
||||
|
||||
# Per-note frontmatter checks
|
||||
template_re = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
for path in all_files:
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
if base in SKIP or template_re.search(path) or not path.endswith(".md"):
|
||||
continue
|
||||
text = texts.get(path) or ""
|
||||
raw, fm = parse_frontmatter(text)
|
||||
if "[[" in raw:
|
||||
flag("frontmatter-wikilink", f"{path}: '[[...]]' inside frontmatter")
|
||||
missing = [k for k in REQUIRED_FM if not str(fm.get(k, "")).strip()]
|
||||
if fm and missing:
|
||||
flag("missing-frontmatter", f"{path}: missing {', '.join(missing)}")
|
||||
# Kind-scoped completeness: entity notes must classify (status + tags). The
|
||||
# per-kind requirement map lives in chorus_index (KIND_REQUIRED_FM) so capture's
|
||||
# defaults, this check, and sweep's backfill all read the same source of truth.
|
||||
kind = idx_mod.kind_for_path(path)
|
||||
if fm and kind:
|
||||
for field in idx_mod.KIND_REQUIRED_FM.get(kind, ()):
|
||||
if not fm_field_populated(raw, fm, field):
|
||||
what = f"missing {field}" if field not in fm else f"empty {field}"
|
||||
flag("incomplete-frontmatter", f"{path}: {what}")
|
||||
created, updated = parse_date(fm.get("created")), parse_date(fm.get("updated"))
|
||||
if created and updated and updated < created:
|
||||
flag("date-order", f"{path}: updated {updated} is before created {created}")
|
||||
if updated and updated > TODAY:
|
||||
flag("future-date", f"{path}: updated {updated} is in the future (today {TODAY})")
|
||||
for source_note in as_list(fm.get("source_notes")):
|
||||
if "[[" in str(source_note):
|
||||
flag("source-notes-wikilink", f"{path}: source_notes contains a wikilink '{source_note}'")
|
||||
|
||||
# Projects: folder<->status, stale active, duplicate slugs
|
||||
slug_homes: dict[str, list[str]] = {}
|
||||
for lifecycle in LIFECYCLES:
|
||||
files, _ = list_dir(f"projects/{lifecycle}")
|
||||
for filename in files:
|
||||
if filename in SKIP or not filename.endswith(".md"):
|
||||
continue
|
||||
slug = filename[:-3]
|
||||
slug_homes.setdefault(slug, []).append(lifecycle)
|
||||
text = texts.get(f"projects/{lifecycle}/{filename}") or ""
|
||||
_, fm = parse_frontmatter(text)
|
||||
status = str(fm.get("status", "")).strip()
|
||||
if status and status != lifecycle:
|
||||
flag("folder/status", f"projects/{lifecycle}/{filename}: status='{status}' but folder='{lifecycle}'")
|
||||
if lifecycle == "active":
|
||||
updated = parse_date(fm.get("updated"))
|
||||
if updated and (TODAY - updated).days > STALE_DAYS:
|
||||
flag("stale-active", f"projects/active/{filename}: updated {updated} ({(TODAY - updated).days}d ago) — consider on-hold/")
|
||||
for slug, homes in slug_homes.items():
|
||||
if len(homes) > 1:
|
||||
flag("duplicate-slug", f"'{slug}' exists in {', '.join(homes)}")
|
||||
|
||||
# Daily notes: duplicate "## Agent Log" headings
|
||||
for path in all_files:
|
||||
if re.match(r"^journal/daily/.*\.md$", path):
|
||||
text = texts.get(path) or ""
|
||||
count = len(re.findall(r"(?m)^## Agent Log\s*$", text))
|
||||
if count > 1:
|
||||
flag("duplicate-agent-log", f"{path}: {count} '## Agent Log' headings")
|
||||
|
||||
# Inbox: captures aging past INBOX_DAYS
|
||||
inbox = get("inbox/captures/inbox.md") or ""
|
||||
for line in inbox.splitlines():
|
||||
match = re.match(r"^\s*-\s*(\d{4}-\d{2}-\d{2})\b", line)
|
||||
date = parse_date(match.group(1)) if match else None
|
||||
if date and (TODAY - date).days > INBOX_DAYS:
|
||||
flag("aging-inbox", f"inbox capture {date} ({(TODAY - date).days}d): {line.strip()[:80]}")
|
||||
|
||||
# Scope freshness (drift detector)
|
||||
context = get("_agent/context/current-context.md")
|
||||
if context is not None:
|
||||
_, fm = parse_frontmatter(context)
|
||||
scope_updated = parse_date(fm.get("scope_updated"))
|
||||
if scope_updated is None:
|
||||
flag("scope-no-timestamp",
|
||||
"_agent/context/current-context.md: no scope_updated frontmatter — scope drift cannot be detected; add it (bootstrap.py) and switch scope via `chorus.py scope set`")
|
||||
else:
|
||||
since = [
|
||||
path for path in all_files
|
||||
if (m := re.match(r"^_agent/sessions/(\d{4}-\d{2}-\d{2})", path))
|
||||
and (d := parse_date(m.group(1))) and d > scope_updated
|
||||
]
|
||||
if len(since) >= SCOPE_STALE_SESSIONS:
|
||||
flag("scope-stale",
|
||||
f"scope set {scope_updated}; {len(since)} session(s) logged since without a switch — confirm it still reflects current work (or run `chorus.py scope set`)")
|
||||
|
||||
# ---- Graph health: broken wikilinks, orphans, index drift ----------------
|
||||
# (md_files / md_set were built up top; reuse the shared `texts` cache.)
|
||||
basemap: dict[str, str] = {}
|
||||
for p in md_files:
|
||||
basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)
|
||||
skip_link = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
|
||||
def resolve_link(target: str):
|
||||
target = target.strip()
|
||||
if not target:
|
||||
return None
|
||||
if "/" in target:
|
||||
cand = target if target.endswith(".md") else target + ".md"
|
||||
return cand if cand in md_set else None
|
||||
return basemap.get(idx_mod.slugify(target))
|
||||
|
||||
inbound: set[str] = set()
|
||||
for path in md_files:
|
||||
if path.rsplit("/", 1)[-1] in SKIP or skip_link.search(path):
|
||||
continue
|
||||
text = texts.get(path) or ""
|
||||
_, body = split_frontmatter(text)
|
||||
for t in links.all_wikilinks(body):
|
||||
tp = resolve_link(t)
|
||||
if tp is None:
|
||||
flag("broken-wikilink", f"{path}: [[{t}]] resolves to no note")
|
||||
else:
|
||||
inbound.add(tp)
|
||||
|
||||
def is_indexable(path: str) -> bool:
|
||||
# READMEs and templates are signposts, not entity notes (sweep skips them too).
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
return (idx_mod.kind_for_path(path) is not None
|
||||
and base not in SKIP and not template_re.search(path))
|
||||
|
||||
# orphan: an entity note with no inbound links and an empty ## Related
|
||||
for path in md_files:
|
||||
if not is_indexable(path) or path in inbound:
|
||||
continue
|
||||
text = texts.get(path) or ""
|
||||
if not links.related_targets(text):
|
||||
flag("orphan", f"{path}: no inbound links and empty ## Related ({idx_mod.kind_for_path(path)}) — link it for recall")
|
||||
|
||||
# index drift: entries pointing at gone files, or indexable notes not yet indexed
|
||||
try:
|
||||
index = idx_mod.load()
|
||||
idx_paths = {e.get("path") for e in index.get("entities", {}).values()}
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
if e.get("path") not in md_set:
|
||||
flag("index-stale", f"index entry '{slug}' -> {e.get('path')} no longer exists (run sweep.py)")
|
||||
for path in md_files:
|
||||
if is_indexable(path) and path not in idx_paths:
|
||||
flag("index-missing", f"{path}: indexable note absent from entities.json (run sweep.py)")
|
||||
except Exception as exc:
|
||||
flag("index-error", f"could not check entity index ({exc})")
|
||||
|
||||
if not violations:
|
||||
print("vault-lint: clean — all invariants hold.")
|
||||
return 0
|
||||
|
||||
print(f"vault-lint: {len(violations)} violation(s) found\n")
|
||||
labels = {
|
||||
"folder/status": "Folder <-> status mismatch",
|
||||
"duplicate-slug": "Duplicate slug across lifecycle folders",
|
||||
"frontmatter-wikilink": "Wikilink in frontmatter (breaks reading view)",
|
||||
"duplicate-agent-log": "Duplicate '## Agent Log' heading",
|
||||
"stale-active": f"Stale active project (updated > {STALE_DAYS}d)",
|
||||
"aging-inbox": f"Inbox capture aging (> {INBOX_DAYS}d)",
|
||||
"unknown-path": "Path matches no route in routing.json",
|
||||
"retired-path": "Write to a retired/dead path",
|
||||
"missing-frontmatter": "Missing required frontmatter field",
|
||||
"incomplete-frontmatter": "Incomplete entity frontmatter (status/tags) — sweep.py --apply backfills",
|
||||
"date-order": "updated earlier than created",
|
||||
"future-date": "updated date is in the future",
|
||||
"source-notes-wikilink": "Wikilink in source_notes (must be plain paths)",
|
||||
"routing-manifest": "routing.json problem",
|
||||
"scope-no-timestamp": "current-context has no scope_updated (drift undetectable)",
|
||||
"scope-stale": f"Scope may have drifted (>= {SCOPE_STALE_SESSIONS} sessions since last switch)",
|
||||
"broken-wikilink": "Wikilink resolves to no note (dead link)",
|
||||
"orphan": "Orphan entity note (no inbound links, empty Related)",
|
||||
"index-stale": "Entity-index entry points at a missing file",
|
||||
"index-missing": "Indexable note absent from the entity index",
|
||||
"index-error": "Entity-index check failed",
|
||||
}
|
||||
grouped: dict[str, list[str]] = {}
|
||||
for check, message in violations:
|
||||
grouped.setdefault(check, []).append(message)
|
||||
for check, messages in grouped.items():
|
||||
print(f"## {labels.get(check, check)}")
|
||||
for message in messages:
|
||||
print(f" - {message}")
|
||||
print()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user