forked from jason/echo
ver 0.9
This commit is contained in:
jason
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""bootstrap.py — stand up (or repair) an ECHO vault deterministically.
|
||||
|
||||
Idempotent and additive: every write is probe-before-write and NEVER overwrites
|
||||
an existing file. The marker (_agent/echo-vault.md) is written LAST, so the vault
|
||||
is only flagged "set up" once every piece is in place. Re-running is the repair
|
||||
path (it fills in only what is missing). Cross-platform: pure Python via echo.py.
|
||||
|
||||
Usage:
|
||||
bootstrap.py [--dry-run]
|
||||
|
||||
Env: ECHO_BASE, ECHO_KEY (via echo.py), ECHO_TODAY (YYYY-MM-DD for {{DATE}}).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
SKILL_DIR = SCRIPT_DIR.parent
|
||||
SCAFFOLD = SKILL_DIR / "scaffold"
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import echo # noqa: E402
|
||||
|
||||
LEAVES = [
|
||||
"inbox/captures", "inbox/imports", "inbox/processing-log",
|
||||
"journal/daily", "journal/weekly", "journal/monthly", "journal/quarterly",
|
||||
"journal/annual", "journal/templates",
|
||||
"projects/active", "projects/incubating", "projects/on-hold", "projects/archived",
|
||||
"areas/business", "areas/personal", "areas/learning", "areas/systems",
|
||||
"resources/concepts", "resources/references", "resources/people",
|
||||
"resources/companies", "resources/meetings",
|
||||
"decisions/by-date",
|
||||
"_agent/context", "_agent/memory/working", "_agent/memory/episodic",
|
||||
"_agent/memory/semantic", "_agent/sessions", "_agent/health", "_agent/templates",
|
||||
"_agent/heartbeat", "_agent/skills/active", "_agent/skills/archived", "_agent/locks",
|
||||
"_agent/index",
|
||||
]
|
||||
|
||||
|
||||
def exists(path: str) -> bool:
|
||||
status, _ = echo.request("GET", echo.vault_url(path))
|
||||
return status == 200
|
||||
|
||||
|
||||
def put_text(path: str, text: str) -> None:
|
||||
status, body = echo.request("PUT", echo.vault_url(path), data=text.encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
echo.check(status, body, f"bootstrap put {path}")
|
||||
|
||||
|
||||
def seed(path: str, source: Path, dry_run: bool) -> None:
|
||||
if exists(path):
|
||||
print(f"bootstrap: skip (exists) {path}")
|
||||
return
|
||||
if dry_run:
|
||||
print(f"bootstrap: would seed {path} <- {source.relative_to(SKILL_DIR)}")
|
||||
return
|
||||
text = source.read_text(encoding="utf-8").replace("{{DATE}}", echo.today())
|
||||
put_text(path, text)
|
||||
print(f"bootstrap: seeded {path}")
|
||||
|
||||
|
||||
def leaf_readme(folder: str, dry_run: bool) -> None:
|
||||
path = f"{folder}/README.md"
|
||||
if exists(path):
|
||||
return
|
||||
if dry_run:
|
||||
print(f"bootstrap: would readme {path}")
|
||||
return
|
||||
name = folder.rsplit("/", 1)[-1]
|
||||
put_text(path, f"# {name}\n\nMemory vault folder. See the echo-memory plugin for conventions.\n")
|
||||
print(f"bootstrap: readme {path}")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Bootstrap or repair an ECHO vault")
|
||||
parser.add_argument("--dry-run", "-n", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if not SCAFFOLD.is_dir():
|
||||
print(f"bootstrap: scaffold not found at {SCAFFOLD}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if exists("_agent/echo-vault.md"):
|
||||
status, body = echo.request("GET", echo.vault_url("_agent/echo-vault.md"))
|
||||
version = "unknown"
|
||||
if status == 200:
|
||||
version = next((ln.split(":", 1)[1].strip()
|
||||
for ln in body.decode(errors="replace").splitlines()
|
||||
if ln.startswith("schema_version:")), "unknown")
|
||||
print(f"bootstrap: marker present (schema_version={version}). Running repair pass (fills only missing files).")
|
||||
|
||||
for folder in LEAVES:
|
||||
leaf_readme(folder, args.dry_run)
|
||||
|
||||
templates = SCAFFOLD / "templates"
|
||||
if templates.is_dir():
|
||||
for source in sorted(templates.rglob("*.md")):
|
||||
seed(source.relative_to(templates).as_posix(), source, args.dry_run)
|
||||
|
||||
seed("_agent/memory/semantic/operator-preferences.md", SCAFFOLD / "anchors" / "operator-preferences.seed.md", args.dry_run)
|
||||
seed("_agent/context/current-context.md", SCAFFOLD / "anchors" / "current-context.seed.md", args.dry_run)
|
||||
seed("inbox/captures/inbox.md", SCAFFOLD / "anchors" / "inbox.seed.md", args.dry_run)
|
||||
seed("README.md", SCAFFOLD / "README.vault.md", args.dry_run)
|
||||
seed("_agent/echo-vault.md", SCAFFOLD / "echo-vault.md", args.dry_run) # marker LAST
|
||||
|
||||
print(f"bootstrap: done ({'DRY-RUN ' if args.dry_run else ''}{echo.today()}).")
|
||||
print("bootstrap: next: create today's daily note + a bootstrap session log + heartbeat (see SKILL.md).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# bootstrap.sh — stand up (or repair) an ECHO vault deterministically.
|
||||
#
|
||||
# Idempotent and additive: every write is probe-before-write and NEVER overwrites an
|
||||
# existing file. The marker (_agent/echo-vault.md) is written LAST, so the vault is
|
||||
# only flagged "set up" once every piece is in place. Safe to re-run any time — that
|
||||
# is also the "repair" path (it fills in only what is missing).
|
||||
#
|
||||
# All scaffold is resolved relative to THIS script's location, so it works regardless
|
||||
# of the caller's CWD (fixes the old `@scaffold/...` relative-path assumption).
|
||||
#
|
||||
# Usage:
|
||||
# bootstrap.sh [--dry-run]
|
||||
#
|
||||
# Env: ECHO_BASE, ECHO_KEY (passed through to echo.sh), ECHO_TODAY (YYYY-MM-DD for {{DATE}}).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SCAFFOLD="$SKILL_DIR/scaffold"
|
||||
ECHO="$SCRIPT_DIR/echo.sh"
|
||||
TODAY="${ECHO_TODAY:-$(date +%Y-%m-%d)}"
|
||||
DRY=0
|
||||
[ "${1:-}" = "--dry-run" ] || [ "${1:-}" = "-n" ] && DRY=1
|
||||
|
||||
[ -d "$SCAFFOLD" ] || { echo "bootstrap: scaffold not found at $SCAFFOLD" >&2; exit 1; }
|
||||
[ -x "$ECHO" ] || chmod +x "$ECHO" 2>/dev/null || true
|
||||
|
||||
say() { echo "bootstrap: $*"; }
|
||||
exists() { ECHO_VERIFY=0 "$ECHO" get "$1" >/dev/null 2>&1; } # 0 = present(200), nonzero = absent/404
|
||||
|
||||
# seed VAULT_PATH from LOCAL_FILE (with {{DATE}} substitution), only if absent
|
||||
seed() {
|
||||
local vpath="$1" local_file="$2"
|
||||
if exists "$vpath"; then say "skip (exists) $vpath"; return 0; fi
|
||||
if [ "$DRY" = "1" ]; then say "would seed $vpath <- ${local_file#$SKILL_DIR/}"; return 0; fi
|
||||
sed "s/{{DATE}}/$TODAY/g" "$local_file" | ECHO_VERIFY=1 "$ECHO" put "$vpath" - >/dev/null
|
||||
say "seeded $vpath"
|
||||
}
|
||||
|
||||
# write a one-line leaf README only if absent
|
||||
leaf_readme() {
|
||||
local dir="$1" name="${1##*/}"
|
||||
local vpath="$dir/README.md"
|
||||
if exists "$vpath"; then return 0; fi
|
||||
if [ "$DRY" = "1" ]; then say "would readme $vpath"; return 0; fi
|
||||
printf '# %s\n\nMemory vault folder. See the echo-memory plugin for conventions.\n' "$name" \
|
||||
| ECHO_VERIFY=0 "$ECHO" put "$vpath" - >/dev/null
|
||||
say "readme $vpath"
|
||||
}
|
||||
|
||||
# ---- Pre-flight: is the vault already bootstrapped? --------------------------
|
||||
if exists "_agent/echo-vault.md"; then
|
||||
ver="$("$ECHO" get _agent/echo-vault.md 2>/dev/null | sed -n 's/^schema_version:[[:space:]]*//p' | head -1)"
|
||||
say "marker present (schema_version=${ver:-unknown}). Running repair pass (fills only missing files)."
|
||||
CUR_SCHEMA=2
|
||||
if [ -n "$ver" ] && [ "$ver" -lt "$CUR_SCHEMA" ] 2>/dev/null; then
|
||||
say "NOTE: schema_version $ver < $CUR_SCHEMA — run migrate.sh before relying on the vault."
|
||||
fi
|
||||
fi
|
||||
|
||||
# ---- 1. Folder tree (leaf READMEs guarantee non-empty dirs) ------------------
|
||||
LEAVES=(
|
||||
inbox/captures inbox/imports inbox/processing-log
|
||||
journal/daily journal/weekly journal/monthly journal/quarterly journal/annual journal/templates
|
||||
projects/active projects/incubating projects/on-hold projects/archived
|
||||
areas/business areas/personal areas/learning areas/systems
|
||||
resources/concepts resources/references resources/people resources/companies resources/meetings
|
||||
decisions/by-date
|
||||
_agent/context _agent/memory/working _agent/memory/episodic _agent/memory/semantic
|
||||
_agent/sessions _agent/health _agent/templates _agent/heartbeat
|
||||
_agent/skills/active _agent/skills/archived _agent/locks
|
||||
)
|
||||
for d in "${LEAVES[@]}"; do leaf_readme "$d"; done
|
||||
|
||||
# ---- 2. Templates (mirror scaffold/templates/ 1:1 into the vault) ------------
|
||||
if [ -d "$SCAFFOLD/templates" ]; then
|
||||
while IFS= read -r f; do
|
||||
rel="${f#$SCAFFOLD/templates/}"
|
||||
seed "$rel" "$f"
|
||||
done < <(find "$SCAFFOLD/templates" -type f -name '*.md')
|
||||
fi
|
||||
|
||||
# ---- 3. Anchor seeds (only if absent — never fabricate facts) ----------------
|
||||
seed "_agent/memory/semantic/operator-preferences.md" "$SCAFFOLD/anchors/operator-preferences.seed.md"
|
||||
seed "_agent/context/current-context.md" "$SCAFFOLD/anchors/current-context.seed.md"
|
||||
seed "inbox/captures/inbox.md" "$SCAFFOLD/anchors/inbox.seed.md"
|
||||
|
||||
# ---- 4. Vault README (human signpost) ----------------------------------------
|
||||
seed "README.md" "$SCAFFOLD/README.vault.md"
|
||||
|
||||
# ---- 5. Marker (write LAST) --------------------------------------------------
|
||||
seed "_agent/echo-vault.md" "$SCAFFOLD/echo-vault.md"
|
||||
|
||||
say "done (${DRY:+DRY-RUN }$TODAY)."
|
||||
say "Next: create today's daily note + a bootstrap session log + heartbeat (see SKILL.md First-run trace)."
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""check_routing.py — keep the routing VIEWS in sync with routing.json.
|
||||
|
||||
routing.json is the canonical route manifest. SKILL.md ("Where to Write"),
|
||||
references/routing-map.md, and references/api-reference.md are hand-maintained
|
||||
human views of it, guarded until now only by a prose "routing.json wins"
|
||||
convention. This check makes that convention mechanically true:
|
||||
|
||||
CHECK A (validity, all three docs): every concrete vault path mentioned in the
|
||||
docs must match some route OR retired pattern in routing.json. Catches
|
||||
a doc pointing at a path the manifest does not define (typo, stale row).
|
||||
|
||||
CHECK B (coverage, routing-map.md): every route in routing.json must be named
|
||||
by at least one path in routing-map.md, which claims to be the complete
|
||||
canonical map. Catches a route added to the manifest but never documented.
|
||||
|
||||
Exit: 0 = in sync, 1 = drift found, 2 = could not read inputs. No network, no vault.
|
||||
Run it in CI / from the eval harness; it is pure local file parsing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
SKILL_DIR = SCRIPT_DIR.parent
|
||||
ROUTING_JSON = SCRIPT_DIR / "routing.json"
|
||||
DOCS = {
|
||||
"SKILL.md": SKILL_DIR / "SKILL.md",
|
||||
"routing-map.md": SKILL_DIR / "references" / "routing-map.md",
|
||||
"api-reference.md": SKILL_DIR / "references" / "api-reference.md",
|
||||
}
|
||||
CANONICAL_MAP = "routing-map.md"
|
||||
|
||||
ROOTS = ("_agent", "inbox", "journal", "projects", "areas", "resources", "decisions")
|
||||
FILE_SUFFIXES = (".md", ".lock", ".json")
|
||||
|
||||
# Order matters: substitute the most specific placeholder first.
|
||||
DATE_SUBS = [
|
||||
("YYYY-MM-DD", "2026-01-02"),
|
||||
("YYYY-Www", "2026-W01"),
|
||||
("YYYY-MM", "2026-01"),
|
||||
("YYYY-Qn", "2026-Q1"),
|
||||
("HHMM", "0900"),
|
||||
("YYYY", "2026"),
|
||||
]
|
||||
|
||||
# Named placeholders whose route constrains the value to a fixed vocabulary, so a
|
||||
# generic "sample" would not match the pattern.
|
||||
NAMED_SUBS = {
|
||||
"<lifecycle>": "active",
|
||||
"<domain>": "business",
|
||||
}
|
||||
|
||||
|
||||
def load_routing():
|
||||
import json
|
||||
data = json.loads(ROUTING_JSON.read_text(encoding="utf-8"))
|
||||
routes = [(r["id"], re.compile(r["pattern"]), stem(r["pattern"])) for r in data.get("routes", [])]
|
||||
retired = [re.compile(r["pattern"]) for r in data.get("retired", [])]
|
||||
return routes, retired
|
||||
|
||||
|
||||
def stem(pattern: str) -> str:
|
||||
"""Literal directory prefix of a regex pattern, e.g. ^projects/active/[^/]+\\.md$ -> projects/active/."""
|
||||
body = pattern[1:] if pattern.startswith("^") else pattern
|
||||
literal = []
|
||||
for ch in body:
|
||||
if ch in "\\([{+*?.|^$":
|
||||
break
|
||||
literal.append(ch)
|
||||
text = "".join(literal)
|
||||
return text[: text.rfind("/") + 1] if "/" in text else ""
|
||||
|
||||
|
||||
def strip_fenced_blocks(text: str) -> str:
|
||||
out, in_fence = [], False
|
||||
for line in text.splitlines():
|
||||
if line.lstrip().startswith("```"):
|
||||
in_fence = not in_fence
|
||||
continue
|
||||
if not in_fence:
|
||||
out.append(line)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def expand_braces(token: str) -> list[str]:
|
||||
"""Expand a single brace group: a/{x,y}/b -> [a/x/b, a/y/b]. Recurses for nested groups."""
|
||||
match = re.search(r"\{([^{}]*)\}", token)
|
||||
if not match:
|
||||
return [token]
|
||||
pre, post = token[: match.start()], token[match.end():]
|
||||
results = []
|
||||
for part in match.group(1).split(","):
|
||||
results.extend(expand_braces(pre + part.strip() + post))
|
||||
return results
|
||||
|
||||
|
||||
def concretize(token: str) -> str:
|
||||
for placeholder, sample in DATE_SUBS:
|
||||
token = token.replace(placeholder, sample)
|
||||
for placeholder, sample in NAMED_SUBS.items():
|
||||
token = token.replace(placeholder, sample)
|
||||
return re.sub(r"<[^>]+>", "sample", token)
|
||||
|
||||
|
||||
def looks_like_path(token: str) -> bool:
|
||||
if " " in token or "::" in token or token.count("`"):
|
||||
return False
|
||||
# README signposts can live under any folder, including placeholder folders.
|
||||
if token == "README.md" or token.endswith("/README.md"):
|
||||
return True
|
||||
# Otherwise must sit under a known vault root directory (root + "/"), which
|
||||
# excludes shorthand like `inbox.md` and non-paths like `application/vnd...`.
|
||||
if not any(token.startswith(root + "/") for root in ROOTS):
|
||||
return False
|
||||
return token.endswith("/") or token.endswith(FILE_SUFFIXES) or "<" in token or "{" in token
|
||||
|
||||
|
||||
def extract_tokens(text: str) -> set[str]:
|
||||
text = strip_fenced_blocks(text)
|
||||
tokens: set[str] = set()
|
||||
for raw in re.findall(r"`([^`]+)`", text):
|
||||
raw = raw.strip()
|
||||
for piece in expand_braces(raw):
|
||||
if looks_like_path(piece):
|
||||
tokens.add(piece)
|
||||
return tokens
|
||||
|
||||
|
||||
def classify(tokens: set[str]):
|
||||
files, dirs = set(), set()
|
||||
for token in tokens:
|
||||
if token.endswith("/"):
|
||||
dirs.add(token)
|
||||
else:
|
||||
files.add(concretize(token))
|
||||
return files, dirs
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
routes, retired = load_routing()
|
||||
except Exception as exc:
|
||||
print(f"check-routing: cannot read routing.json ({exc})", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
problems: list[str] = []
|
||||
doc_tokens: dict[str, tuple[set[str], set[str]]] = {}
|
||||
for name, path in DOCS.items():
|
||||
try:
|
||||
files, dirs = classify(extract_tokens(path.read_text(encoding="utf-8")))
|
||||
except Exception as exc:
|
||||
print(f"check-routing: cannot read {name} ({exc})", file=sys.stderr)
|
||||
return 2
|
||||
doc_tokens[name] = (files, dirs)
|
||||
|
||||
route_stems = {st for _, _, st in routes if st}
|
||||
|
||||
# CHECK A — every documented path is valid under the manifest.
|
||||
for name, (files, dirs) in doc_tokens.items():
|
||||
for f in sorted(files):
|
||||
if not (any(rx.match(f) for _, rx, _ in routes) or any(rx.match(f) for rx in retired)):
|
||||
problems.append(f"[A] {name}: path `{f}` matches no route or retired pattern in routing.json")
|
||||
for d in sorted(dirs):
|
||||
ok = (any(rx.match(d) for rx in retired)
|
||||
or any(st == d or st.startswith(d) or d.startswith(st) for st in route_stems))
|
||||
if not ok:
|
||||
problems.append(f"[A] {name}: folder `{d}` matches no route stem or retired pattern")
|
||||
|
||||
# CHECK B — every route appears in the canonical map.
|
||||
map_files, map_dirs = doc_tokens[CANONICAL_MAP]
|
||||
for rid, rx, st in routes:
|
||||
covered = any(rx.match(f) for f in map_files) or (st and st in map_dirs)
|
||||
if not covered:
|
||||
problems.append(f"[B] route '{rid}' ({rx.pattern}) is not documented in {CANONICAL_MAP}")
|
||||
|
||||
if problems:
|
||||
print(f"check-routing: {len(problems)} drift issue(s) found\n")
|
||||
for p in problems:
|
||||
print(f" - {p}")
|
||||
return 1
|
||||
print(f"check-routing: in sync — {len(routes)} routes documented and all doc paths are valid.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,521 @@
|
||||
#!/usr/bin/env python3
|
||||
"""echo.py — the single validated, cross-platform client for the ECHO Obsidian
|
||||
Local REST API.
|
||||
|
||||
Replaces the original echo.sh so the toolchain runs identically on Windows,
|
||||
macOS, and Linux with no bash / GNU-vs-BSD `date` dependency — only a Python 3
|
||||
interpreter (which the linter already requires).
|
||||
|
||||
Every read/write to the vault should go through this script. It centralizes:
|
||||
auth injection, HTTP-status checking (non-zero exit on >=400 so a failed write
|
||||
can never look like success), one bounded retry on 5xx/connection errors,
|
||||
WHOLE-LINE idempotent append (a new line that is merely a substring of an
|
||||
existing one is no longer wrongly skipped), correct `::` heading-target handling,
|
||||
frontmatter field patches, a read-back-confirmed advisory multi-writer lock, a
|
||||
one-call cold-start `load`, and scope show/set.
|
||||
|
||||
Config (env overrides; defaults match the rest of the plugin):
|
||||
ECHO_BASE default https://echoapi.alwisp.com
|
||||
ECHO_KEY default the plugin bearer token (env overrides it)
|
||||
ECHO_VERIFY default 1 — read-back verify after a PUT
|
||||
ECHO_LOCK_TTL default 900 — seconds before an advisory lock is considered stale
|
||||
ECHO_TODAY YYYY-MM-DD to use as "today" (pass the conversation's currentDate)
|
||||
|
||||
Usage:
|
||||
echo.py load # cold-start: the 6 orientation reads in one call
|
||||
echo.py get <path> # print file contents (404 -> exit 44)
|
||||
echo.py map <path> # document-map JSON (headings/blocks/frontmatter)
|
||||
echo.py ls <dir> # directory listing JSON
|
||||
echo.py search <query...> # /search/simple
|
||||
echo.py put <path> [file] # create/overwrite (body from file or stdin)
|
||||
echo.py post <path> [file] # raw append (NON-idempotent; prefer `append`)
|
||||
echo.py append <path> <line> # idempotent append: skips only on an exact whole-line match
|
||||
echo.py patch <path> <append|prepend|replace> <heading|frontmatter|block> <target> [file]
|
||||
echo.py fm <path> <field> <json-value> # PATCH a frontmatter scalar
|
||||
echo.py bump <path> [YYYY-MM-DD] # set frontmatter updated: to today (or given date)
|
||||
echo.py delete <path> # DELETE (destructive; explicit use only)
|
||||
echo.py lock <owner-id> # acquire advisory lock (exit 75 if held & fresh, or lost the race)
|
||||
echo.py unlock <owner-id> # release advisory lock if owned by <owner-id>
|
||||
echo.py scope show # print active scope, its freshness, and sessions-since
|
||||
echo.py scope set "<text>" # switch scope atomically (history + replace + stamp scope_updated)
|
||||
|
||||
Exit codes: 0 ok · 44 not-found(404) · 75 lock-held/lost · 2 usage · 1 other HTTP/transport error.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import locale
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
# Never let a non-ASCII byte (em-dash in our own output, or Unicode in a fetched
|
||||
# note) raise UnicodeEncodeError on a legacy Windows console.
|
||||
for _stream in (sys.stdout, sys.stderr):
|
||||
try:
|
||||
_stream.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
BASE = os.environ.get("ECHO_BASE", "https://echoapi.alwisp.com").rstrip("/")
|
||||
# The token stays hardcoded for this personal plugin; ECHO_KEY env still overrides.
|
||||
DEFAULT_KEY = "241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab"
|
||||
KEY = os.environ.get("ECHO_KEY") or DEFAULT_KEY
|
||||
VERIFY = os.environ.get("ECHO_VERIFY", "1") != "0"
|
||||
LOCK_TTL = int(os.environ.get("ECHO_LOCK_TTL", "900"))
|
||||
|
||||
LOCK_PATH = "_agent/locks/vault.lock"
|
||||
|
||||
|
||||
class EchoError(RuntimeError):
|
||||
def __init__(self, message: str, code: int = 1) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
def today() -> str:
|
||||
return os.environ.get("ECHO_TODAY") or dt.date.today().isoformat()
|
||||
|
||||
|
||||
def now_iso() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def vault_url(path: str) -> str:
|
||||
safe = "/".join(urllib.parse.quote(part) for part in path.split("/"))
|
||||
if path.endswith("/") and not safe.endswith("/"):
|
||||
safe += "/"
|
||||
return f"{BASE}/vault/{safe}"
|
||||
|
||||
|
||||
def request(method: str, url: str, data: bytes | None = None,
|
||||
headers: dict[str, str] | None = None) -> tuple[int, bytes]:
|
||||
"""One bounded retry on transport failure (status 0) or 5xx. Returns (status, body)."""
|
||||
merged = {"Authorization": f"Bearer {KEY}", **(headers or {})}
|
||||
last_status, last_body = 0, b""
|
||||
for attempt in range(2):
|
||||
req = urllib.request.Request(url, data=data, method=method, headers=merged)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as response:
|
||||
return response.status, response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = exc.read()
|
||||
if exc.code >= 500 and attempt == 0:
|
||||
time.sleep(1)
|
||||
continue
|
||||
return exc.code, body
|
||||
except Exception as exc: # connection error, timeout, DNS, ...
|
||||
last_status, last_body = 0, str(exc).encode()
|
||||
if attempt == 0:
|
||||
time.sleep(1)
|
||||
continue
|
||||
return last_status, last_body
|
||||
|
||||
|
||||
def check(status: int, body: bytes, context: str) -> None:
|
||||
if status == 404:
|
||||
raise EchoError(f"{context}: not found", 44)
|
||||
if status == 0:
|
||||
raise EchoError(f"{context}: vault unreachable ({body.decode(errors='replace')}) [{BASE}]", 1)
|
||||
if status >= 400:
|
||||
raise EchoError(f"{context}: HTTP {status} - {body.decode(errors='replace')}", 1)
|
||||
|
||||
|
||||
def read_body(arg: str | None) -> bytes:
|
||||
if arg in (None, "-"):
|
||||
raw = sys.stdin.buffer.read()
|
||||
try:
|
||||
text = raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode(locale.getpreferredencoding(False) or "cp1252", errors="replace")
|
||||
return text.encode("utf-8")
|
||||
return Path(arg).read_bytes()
|
||||
|
||||
|
||||
def normalize_patch_body(data: bytes, op: str, target_type: str) -> bytes:
|
||||
"""Heading PATCH bodies need a leading newline on replace and a trailing
|
||||
newline on append/prepend so the API does not concatenate onto adjacent text."""
|
||||
if target_type != "heading":
|
||||
return data
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
if op == "replace":
|
||||
text = text.strip("\r\n")
|
||||
text = f"\n{text}\n" if text else "\n"
|
||||
elif text and not text.endswith("\n"):
|
||||
text += "\n"
|
||||
return text.encode("utf-8")
|
||||
|
||||
|
||||
def normalize_json_scalar(value: str) -> bytes:
|
||||
"""Accept either a raw scalar ('Fabrication Lead') or pre-formed JSON ('"2026-06-20"')."""
|
||||
value = value.strip()
|
||||
try:
|
||||
json.loads(value)
|
||||
return value.encode("utf-8")
|
||||
except json.JSONDecodeError:
|
||||
return json.dumps(value).encode("utf-8")
|
||||
|
||||
|
||||
def temp_file(data: bytes) -> str:
|
||||
fh = tempfile.NamedTemporaryFile(delete=False)
|
||||
try:
|
||||
fh.write(data)
|
||||
return fh.name
|
||||
finally:
|
||||
fh.close()
|
||||
|
||||
|
||||
# ---- commands ----------------------------------------------------------------
|
||||
|
||||
def cmd_get(path: str) -> int:
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"get {path}")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_map(path: str) -> int:
|
||||
status, body = request("GET", vault_url(path),
|
||||
headers={"Accept": "application/vnd.olrapi.document-map+json"})
|
||||
check(status, body, f"map {path}")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_ls(path: str) -> int:
|
||||
if not path.endswith("/"):
|
||||
path += "/"
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"ls {path}")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_search(query: list[str]) -> int:
|
||||
q = urllib.parse.quote_plus(" ".join(query))
|
||||
status, body = request("POST", f"{BASE}/search/simple/?query={q}")
|
||||
check(status, body, "search")
|
||||
sys.stdout.buffer.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_put(path: str, file_arg: str | None) -> int:
|
||||
data = read_body(file_arg)
|
||||
status, body = request("PUT", vault_url(path), data=data,
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
check(status, body, f"put {path}")
|
||||
if VERIFY:
|
||||
status, _ = request("GET", vault_url(path))
|
||||
if status != 200:
|
||||
raise EchoError(f"put {path}: write did not verify (GET returned {status})")
|
||||
print(f"ok: PUT {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_post(path: str, file_arg: str | None) -> int:
|
||||
data = read_body(file_arg)
|
||||
status, body = request("POST", vault_url(path), data=data,
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
check(status, body, f"post {path}")
|
||||
print(f"ok: POST {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_append(path: str, line: str) -> int:
|
||||
"""Idempotent append. Skips ONLY when the exact whole line already exists —
|
||||
a new line that is a substring of an existing one is still appended."""
|
||||
status, body = request("GET", vault_url(path))
|
||||
if status == 200:
|
||||
existing = [ln.rstrip("\r") for ln in body.decode(errors="replace").splitlines()]
|
||||
if line.rstrip("\r") in existing:
|
||||
print(f"skip: line already present in {path}")
|
||||
return 0
|
||||
elif status != 404:
|
||||
check(status, body, f"append(read) {path}")
|
||||
status, body = request("POST", vault_url(path), data=f"{line}\n".encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
check(status, body, f"append {path}")
|
||||
print(f"ok: APPEND {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_patch(path: str, op: str, target_type: str, target: str, file_arg: str | None) -> int:
|
||||
if op not in {"append", "prepend", "replace"}:
|
||||
raise EchoError("patch: op must be append, prepend, or replace", 2)
|
||||
if target_type not in {"heading", "frontmatter", "block"}:
|
||||
raise EchoError("patch: target-type must be heading, frontmatter, or block", 2)
|
||||
data = normalize_patch_body(read_body(file_arg), op, target_type)
|
||||
status, body = request(
|
||||
"PATCH",
|
||||
vault_url(path),
|
||||
data=data,
|
||||
headers={
|
||||
"Operation": op,
|
||||
"Target-Type": target_type,
|
||||
"Target": target,
|
||||
"Content-Type": "application/json" if target_type == "frontmatter" else "text/markdown",
|
||||
},
|
||||
)
|
||||
check(status, body, f"patch {path} ({target})")
|
||||
print(f"ok: PATCH {op} {target_type} '{target}' -> {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_fm(path: str, field: str, value: str) -> int:
|
||||
return cmd_patch(path, "replace", "frontmatter", field, temp_file(normalize_json_scalar(value)))
|
||||
|
||||
|
||||
def cmd_delete(path: str) -> int:
|
||||
status, body = request("DELETE", vault_url(path))
|
||||
check(status, body, f"delete {path}")
|
||||
print(f"ok: DELETE {path}")
|
||||
return 0
|
||||
|
||||
|
||||
def parse_lock_time(value: str) -> int:
|
||||
try:
|
||||
return int(dt.datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ")
|
||||
.replace(tzinfo=dt.timezone.utc).timestamp())
|
||||
except Exception:
|
||||
# Unparseable timestamp -> treat the lock as FRESH (return "now"), so a
|
||||
# garbled lock is respected rather than silently stomped.
|
||||
return int(time.time())
|
||||
|
||||
|
||||
def cmd_lock(owner: str) -> int:
|
||||
status, body = request("GET", vault_url(LOCK_PATH))
|
||||
if status == 200 and body.strip():
|
||||
current = body.decode(errors="replace").strip()
|
||||
held_owner, _, held_iso = current.partition(" @ ")
|
||||
if held_owner != owner and int(time.time()) - parse_lock_time(held_iso) < LOCK_TTL:
|
||||
print(f"lock held by '{held_owner}' since {held_iso} (fresh)", file=sys.stderr)
|
||||
return 75
|
||||
status, body = request("PUT", vault_url(LOCK_PATH), data=f"{owner} @ {now_iso()}\n".encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
check(status, body, "lock")
|
||||
# Read-back confirm we actually hold it. The REST API has no compare-and-swap,
|
||||
# so two racers can both PUT; last writer wins. Whoever does not own the file
|
||||
# on read-back backs off instead of proceeding under a false lock.
|
||||
status, body = request("GET", vault_url(LOCK_PATH))
|
||||
if status == 200:
|
||||
held_owner = body.decode(errors="replace").split(" @ ", 1)[0].strip()
|
||||
if held_owner != owner:
|
||||
print(f"lock race: now held by '{held_owner}', not '{owner}' — backing off", file=sys.stderr)
|
||||
return 75
|
||||
print(f"ok: locked by {owner}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_unlock(owner: str) -> int:
|
||||
status, body = request("GET", vault_url(LOCK_PATH))
|
||||
if status == 200:
|
||||
held_owner = body.decode(errors="replace").split(" @ ", 1)[0].strip()
|
||||
if held_owner != owner:
|
||||
print(f"lock owned by '{held_owner}', not '{owner}' — not releasing", file=sys.stderr)
|
||||
return 75
|
||||
status, body = request("DELETE", vault_url(LOCK_PATH))
|
||||
if status != 404:
|
||||
check(status, body, "unlock")
|
||||
print("ok: unlocked")
|
||||
return 0
|
||||
|
||||
|
||||
def extract_heading(markdown: str, heading: str) -> str:
|
||||
"""Return the body under a `## <heading>` up to the next `## ` heading."""
|
||||
out: list[str] = []
|
||||
capture = False
|
||||
marker = f"## {heading}"
|
||||
for line in markdown.splitlines():
|
||||
if line.strip() == marker:
|
||||
capture = True
|
||||
continue
|
||||
if capture and line.startswith("## "):
|
||||
break
|
||||
if capture:
|
||||
out.append(line)
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
def cmd_scope(subcommand: str, text: str | None = None) -> int:
|
||||
path = "_agent/context/current-context.md"
|
||||
status, body = request("GET", vault_url(path))
|
||||
check(status, body, f"scope {subcommand}")
|
||||
current = body.decode(errors="replace")
|
||||
|
||||
if subcommand == "show":
|
||||
print("-- Active scope --")
|
||||
print(extract_heading(current, "Scope"))
|
||||
scope_updated = ""
|
||||
for line in current.splitlines():
|
||||
if line.startswith("scope_updated:"):
|
||||
scope_updated = line.split(":", 1)[1].strip().strip('"').strip("'")
|
||||
break
|
||||
print(f"scope_updated: {scope_updated or '<missing — drift cannot be detected; run scope set or repair>'}")
|
||||
# Count session logs dated after scope_updated (the drift signal).
|
||||
if scope_updated:
|
||||
status, body = request("GET", vault_url("_agent/sessions/"))
|
||||
if status == 200:
|
||||
try:
|
||||
files = json.loads(body).get("files", [])
|
||||
n = sum(1 for f in files if f.endswith(".md") and f[:10] > scope_updated)
|
||||
print(f"sessions logged since: {n}")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
if subcommand != "set":
|
||||
raise EchoError("scope: use 'show' or 'set \"<text>\"'", 2)
|
||||
if not text:
|
||||
raise EchoError("scope set needs the new scope text", 2)
|
||||
prior = extract_heading(current, "Scope").replace("\n", " ").strip()[:140] or "(prior scope)"
|
||||
cmd_patch(path, "prepend", "heading", "Current Context::Scope History",
|
||||
temp_file(f"- {today()}: {prior}\n".encode()))
|
||||
cmd_patch(path, "replace", "heading", "Current Context::Scope",
|
||||
temp_file(f"{text}\n".encode()))
|
||||
try:
|
||||
cmd_patch(path, "replace", "frontmatter", "scope_updated",
|
||||
temp_file(json.dumps(today()).encode()))
|
||||
except EchoError as exc:
|
||||
raise EchoError(
|
||||
"scope set: body switched, but scope_updated frontmatter is missing "
|
||||
f"(run bootstrap.py to add it) [{exc}]"
|
||||
)
|
||||
print(f"ok: scope switched (prior archived to Scope History; scope_updated={today()})")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_load() -> int:
|
||||
"""Cold-start orientation: the canonical 6 reads in one call. 404s on today's
|
||||
daily note and the inbox are normal (printed as absent, not errors)."""
|
||||
targets = [
|
||||
("marker", "_agent/echo-vault.md"),
|
||||
("preferences", "_agent/memory/semantic/operator-preferences.md"),
|
||||
("context", "_agent/context/current-context.md"),
|
||||
("heartbeat", "_agent/heartbeat/last-session.md"),
|
||||
("today", f"journal/daily/{today()}.md"),
|
||||
("inbox", "inbox/captures/inbox.md"),
|
||||
]
|
||||
marker_missing = False
|
||||
for label, path in targets:
|
||||
status, body = request("GET", vault_url(path))
|
||||
print(f"===== {label}: {path} (HTTP {status}) =====")
|
||||
if status == 200:
|
||||
text = body.decode(errors="replace")
|
||||
sys.stdout.write(text)
|
||||
if not text.endswith("\n"):
|
||||
sys.stdout.write("\n")
|
||||
elif status == 404:
|
||||
print("(absent — fine)")
|
||||
if label == "marker":
|
||||
marker_missing = True
|
||||
else:
|
||||
print(f"(error HTTP {status}: {body.decode(errors='replace')[:200]})")
|
||||
print()
|
||||
if marker_missing:
|
||||
print("NOTE: marker absent — vault not bootstrapped. Run bootstrap.py before relying on memory.")
|
||||
return 0
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Validated cross-platform ECHO vault client")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("load")
|
||||
sub.add_parser("get").add_argument("path")
|
||||
sub.add_parser("map").add_argument("path")
|
||||
sub.add_parser("ls").add_argument("path")
|
||||
sub.add_parser("search").add_argument("query", nargs="+")
|
||||
p = sub.add_parser("put"); p.add_argument("path"); p.add_argument("file", nargs="?")
|
||||
p = sub.add_parser("post"); p.add_argument("path"); p.add_argument("file", nargs="?")
|
||||
p = sub.add_parser("append"); p.add_argument("path"); p.add_argument("line")
|
||||
p = sub.add_parser("patch")
|
||||
p.add_argument("path"); p.add_argument("op"); p.add_argument("target_type")
|
||||
p.add_argument("target"); p.add_argument("file", nargs="?")
|
||||
p = sub.add_parser("fm"); p.add_argument("path"); p.add_argument("field"); p.add_argument("value")
|
||||
p = sub.add_parser("bump"); p.add_argument("path"); p.add_argument("date", nargs="?")
|
||||
sub.add_parser("delete").add_argument("path")
|
||||
sub.add_parser("lock").add_argument("owner")
|
||||
sub.add_parser("unlock").add_argument("owner")
|
||||
p = sub.add_parser("scope")
|
||||
p.add_argument("subcommand", nargs="?", default="show", choices=["show", "set"])
|
||||
p.add_argument("text", nargs="?")
|
||||
|
||||
# High-level operations (entity index + cross-links + recall). See echo_ops.py.
|
||||
sub.add_parser("resolve").add_argument("mention", nargs="+")
|
||||
p = sub.add_parser("recall"); p.add_argument("query", nargs="+"); p.add_argument("--limit", type=int, default=6)
|
||||
p = sub.add_parser("link"); p.add_argument("a"); p.add_argument("b")
|
||||
p = sub.add_parser("capture")
|
||||
p.add_argument("title")
|
||||
p.add_argument("file", nargs="?")
|
||||
p.add_argument("--kind")
|
||||
p.add_argument("--inbox", action="store_true")
|
||||
p.add_argument("--status", default="")
|
||||
p.add_argument("--aliases", default="")
|
||||
p.add_argument("--source", default="")
|
||||
p.add_argument("--date")
|
||||
p.add_argument("--domain", default="business")
|
||||
p.add_argument("--no-log", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
if args.cmd == "load":
|
||||
return cmd_load()
|
||||
if args.cmd == "get":
|
||||
return cmd_get(args.path)
|
||||
if args.cmd == "map":
|
||||
return cmd_map(args.path)
|
||||
if args.cmd == "ls":
|
||||
return cmd_ls(args.path)
|
||||
if args.cmd == "search":
|
||||
return cmd_search(args.query)
|
||||
if args.cmd == "put":
|
||||
return cmd_put(args.path, args.file)
|
||||
if args.cmd == "post":
|
||||
return cmd_post(args.path, args.file)
|
||||
if args.cmd == "append":
|
||||
return cmd_append(args.path, args.line)
|
||||
if args.cmd == "patch":
|
||||
return cmd_patch(args.path, args.op, args.target_type, args.target, args.file)
|
||||
if args.cmd == "fm":
|
||||
return cmd_fm(args.path, args.field, args.value)
|
||||
if args.cmd == "bump":
|
||||
return cmd_fm(args.path, "updated", json.dumps(args.date or today()))
|
||||
if args.cmd == "delete":
|
||||
return cmd_delete(args.path)
|
||||
if args.cmd == "lock":
|
||||
return cmd_lock(args.owner)
|
||||
if args.cmd == "unlock":
|
||||
return cmd_unlock(args.owner)
|
||||
if args.cmd == "scope":
|
||||
return cmd_scope(args.subcommand, args.text)
|
||||
if args.cmd in ("resolve", "recall", "link", "capture"):
|
||||
import echo_ops # lazy: only the high-level ops pull in the index/link modules
|
||||
if args.cmd == "resolve":
|
||||
return echo_ops.resolve(" ".join(args.mention))
|
||||
if args.cmd == "recall":
|
||||
return echo_ops.recall(args.query, limit=args.limit)
|
||||
if args.cmd == "link":
|
||||
return echo_ops.link(args.a, args.b)
|
||||
return echo_ops.capture(
|
||||
args.kind, args.title, args.file, status_v=args.status,
|
||||
aliases=args.aliases.split(",") if args.aliases else [],
|
||||
sources=args.source.split(",") if args.source else [],
|
||||
date=args.date, domain=args.domain, inbox=args.inbox, no_log=args.no_log)
|
||||
except EchoError as exc:
|
||||
print(f"echo.py: {exc}", file=sys.stderr)
|
||||
return exc.code
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,245 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# echo.sh — the single validated client for the ECHO Obsidian Local REST API.
|
||||
#
|
||||
# Every read/write to the vault should go through this script instead of hand-built
|
||||
# curl. It centralizes: auth injection, HTTP-status checking (non-zero exit on >=400
|
||||
# so a failed write can never look like success), one bounded retry on 5xx/connection
|
||||
# errors, idempotent append (read-before-POST), correct `::` heading-target handling,
|
||||
# frontmatter field patches, and an advisory multi-writer lock.
|
||||
#
|
||||
# Config (env overrides; defaults match the rest of the plugin):
|
||||
# ECHO_BASE default https://echoapi.alwisp.com
|
||||
# ECHO_KEY default the plugin bearer token
|
||||
# ECHO_VERIFY default 1 — read-back verify after a PUT
|
||||
# ECHO_LOCK_TTL default 900 — seconds before an advisory lock is considered stale
|
||||
#
|
||||
# Usage:
|
||||
# echo.sh get <path> # print file contents (404 -> exit 44)
|
||||
# echo.sh map <path> # document-map JSON (headings/blocks/frontmatter)
|
||||
# echo.sh ls <dir> # directory listing JSON
|
||||
# echo.sh search <query...> # /search/simple
|
||||
# echo.sh put <path> [file] # create/overwrite (body from file or stdin)
|
||||
# echo.sh post <path> [file] # raw append (NON-idempotent; prefer `append`)
|
||||
# echo.sh append <path> <line> # idempotent append: skips if the exact line exists
|
||||
# echo.sh patch <path> <append|prepend|replace> <heading|frontmatter|block> <target> [file]
|
||||
# echo.sh fm <path> <field> <json-value> # PATCH a frontmatter scalar (e.g. fm p.md updated '"2026-06-19"')
|
||||
# echo.sh bump <path> [YYYY-MM-DD] # set frontmatter updated: to today (or given date)
|
||||
# echo.sh delete <path> # DELETE (destructive; explicit use only)
|
||||
# echo.sh lock <owner-id> # acquire advisory lock (exit 75 if held by someone else & fresh)
|
||||
# echo.sh unlock <owner-id> # release advisory lock if owned by <owner-id>
|
||||
# echo.sh scope show # print active scope, its freshness, and sessions-since
|
||||
# echo.sh scope set "<text>" # switch scope atomically (history + replace + stamp scope_updated)
|
||||
#
|
||||
# Exit codes: 0 ok · 44 not-found(404) · 75 lock-held · 2 usage · 1 other HTTP/transport error.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ECHO_BASE="${ECHO_BASE:-https://echoapi.alwisp.com}"
|
||||
ECHO_BASE="${ECHO_BASE%/}"
|
||||
ECHO_KEY="${ECHO_KEY:-241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab}"
|
||||
ECHO_VERIFY="${ECHO_VERIFY:-1}"
|
||||
ECHO_LOCK_TTL="${ECHO_LOCK_TTL:-900}"
|
||||
AUTH="Authorization: Bearer ${ECHO_KEY}"
|
||||
|
||||
# response-body scratch file (filled by every _curl; read by callers via $RESP)
|
||||
RESP="$(mktemp)"; BODY=""
|
||||
cleanup() { rm -f "$RESP" "${BODY:-}"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
die() { echo "echo.sh: $*" >&2; exit 1; }
|
||||
usage() { sed -n '2,40p' "$0" >&2; exit 2; }
|
||||
|
||||
_today() { echo "${ECHO_TODAY:-$(date +%Y-%m-%d)}"; }
|
||||
_now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
_vault_url() { echo "${ECHO_BASE}/vault/$1"; }
|
||||
|
||||
# _curl METHOD URL [extra curl args...]
|
||||
# Writes the response body to $RESP and the status code to $HTTP — BOTH IN THE PARENT
|
||||
# SHELL (never call this in $(...) or on the right of a pipe, or those globals are lost).
|
||||
# One bounded retry on transport failure (000) or 5xx.
|
||||
HTTP=""
|
||||
_curl() {
|
||||
local method="$1" url="$2"; shift 2
|
||||
local code attempt=0
|
||||
while :; do
|
||||
code="$(curl -sS -X "$method" -H "$AUTH" -o "$RESP" -w '%{http_code}' "$@" "$url" 2>/dev/null || echo 000)"
|
||||
if { [ "$code" = "000" ] || [ "${code:0:1}" = "5" ]; } && [ "$attempt" -lt 1 ]; then
|
||||
attempt=$((attempt+1)); sleep 1; continue
|
||||
fi
|
||||
break
|
||||
done
|
||||
HTTP="$code"
|
||||
}
|
||||
|
||||
# assert $HTTP is acceptable; 404 -> exit 44, other >=400 -> exit 1
|
||||
_check() {
|
||||
local ctx="$1"
|
||||
[ "$HTTP" = "404" ] && exit 44
|
||||
[ "$HTTP" = "000" ] && die "$ctx: vault unreachable (connection failed) [$ECHO_BASE]"
|
||||
[ "${HTTP:-000}" -ge 400 ] && die "$ctx: HTTP $HTTP — $(cat "$RESP")"
|
||||
return 0
|
||||
}
|
||||
|
||||
# capture a body argument (file path or '-'/empty for stdin) into $BODY (a temp file)
|
||||
_capture_body() {
|
||||
BODY="$(mktemp)"
|
||||
if [ "${1:-}" = "" ] || [ "${1:-}" = "-" ]; then cat > "$BODY"; else cat "$1" > "$BODY"; fi
|
||||
}
|
||||
|
||||
cmd="${1:-}"; shift || true
|
||||
case "$cmd" in
|
||||
get)
|
||||
[ $# -ge 1 ] || usage
|
||||
_curl GET "$(_vault_url "$1")"; _check "get $1"; cat "$RESP" ;;
|
||||
|
||||
map)
|
||||
[ $# -ge 1 ] || usage
|
||||
_curl GET "$(_vault_url "$1")" -H 'Accept: application/vnd.olrapi.document-map+json'
|
||||
_check "map $1"; cat "$RESP" ;;
|
||||
|
||||
ls)
|
||||
[ $# -ge 1 ] || usage
|
||||
p="$1"; [ "${p%/}" = "$p" ] && p="$p/"
|
||||
_curl GET "$(_vault_url "$p")"; _check "ls $1"; cat "$RESP" ;;
|
||||
|
||||
search)
|
||||
[ $# -ge 1 ] || usage
|
||||
q="$*"; q="${q// /+}"
|
||||
_curl POST "${ECHO_BASE}/search/simple/?query=${q}"; _check "search"; cat "$RESP" ;;
|
||||
|
||||
put)
|
||||
[ $# -ge 1 ] || usage
|
||||
path="$1"; _capture_body "${2:-}"
|
||||
_curl PUT "$(_vault_url "$path")" -H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "put $path"
|
||||
if [ "$ECHO_VERIFY" = "1" ]; then
|
||||
_curl GET "$(_vault_url "$path")"
|
||||
[ "$HTTP" = "200" ] || die "put $path: write did not verify (GET returned $HTTP)"
|
||||
fi
|
||||
echo "ok: PUT $path" ;;
|
||||
|
||||
post)
|
||||
[ $# -ge 1 ] || usage
|
||||
path="$1"; _capture_body "${2:-}"
|
||||
_curl POST "$(_vault_url "$path")" -H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "post $path"; echo "ok: POST $path" ;;
|
||||
|
||||
append)
|
||||
# idempotent: GET the file, skip the POST if the exact line is already present.
|
||||
[ $# -ge 2 ] || usage
|
||||
path="$1"; line="$2"
|
||||
_curl GET "$(_vault_url "$path")"
|
||||
if [ "$HTTP" = "200" ] && grep -qF -- "$line" "$RESP"; then
|
||||
echo "skip: line already present in $path"; exit 0
|
||||
fi
|
||||
[ "$HTTP" = "200" ] || [ "$HTTP" = "404" ] || _check "append(read) $path"
|
||||
BODY="$(mktemp)"; printf '%s\n' "$line" > "$BODY"
|
||||
_curl POST "$(_vault_url "$path")" -H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "append $path"; echo "ok: APPEND $path" ;;
|
||||
|
||||
patch)
|
||||
[ $# -ge 4 ] || usage
|
||||
path="$1"; op="$2"; ttype="$3"; target="$4"; _capture_body "${5:-}"
|
||||
case "$op" in append|prepend|replace) ;; *) die "patch: op must be append|prepend|replace";; esac
|
||||
case "$ttype" in heading|frontmatter|block) ;; *) die "patch: target-type must be heading|frontmatter|block";; esac
|
||||
_curl PATCH "$(_vault_url "$path")" \
|
||||
-H "Operation: $op" -H "Target-Type: $ttype" -H "Target: $target" \
|
||||
-H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "patch $path ($target)"
|
||||
echo "ok: PATCH $op $ttype '$target' -> $path" ;;
|
||||
|
||||
fm)
|
||||
[ $# -ge 3 ] || usage
|
||||
path="$1"; field="$2"; value="$3"
|
||||
BODY="$(mktemp)"; printf '%s' "$value" > "$BODY"
|
||||
_curl PATCH "$(_vault_url "$path")" \
|
||||
-H 'Operation: replace' -H 'Target-Type: frontmatter' -H "Target: $field" \
|
||||
-H 'Content-Type: application/json' --data-binary @"$BODY"
|
||||
_check "fm $path.$field"; echo "ok: frontmatter $field -> $path" ;;
|
||||
|
||||
bump)
|
||||
[ $# -ge 1 ] || usage
|
||||
path="$1"; d="${2:-$(_today)}"
|
||||
exec "$0" fm "$path" updated "\"$d\"" ;;
|
||||
|
||||
delete)
|
||||
[ $# -ge 1 ] || usage
|
||||
_curl DELETE "$(_vault_url "$1")"; _check "delete $1"; echo "ok: DELETE $1" ;;
|
||||
|
||||
lock)
|
||||
# advisory lock: _agent/locks/vault.lock holds "<owner> @ <iso>". Honored cooperatively.
|
||||
[ $# -ge 1 ] || usage
|
||||
owner="$1"; lockpath="_agent/locks/vault.lock"
|
||||
_curl GET "$(_vault_url "$lockpath")"
|
||||
if [ "$HTTP" = "200" ] && [ -s "$RESP" ]; then
|
||||
cur="$(cat "$RESP")"; held_owner="${cur%% @ *}"; held_iso="${cur##* @ }"; held_iso="${held_iso%$'\n'}"
|
||||
held_epoch="$(date -u -j -f '%Y-%m-%dT%H:%M:%SZ' "$held_iso" +%s 2>/dev/null \
|
||||
|| date -u -d "$held_iso" +%s 2>/dev/null || echo 0)"
|
||||
now_epoch="$(date -u +%s)"
|
||||
if [ "$held_owner" != "$owner" ] && [ $((now_epoch - held_epoch)) -lt "$ECHO_LOCK_TTL" ]; then
|
||||
echo "lock held by '$held_owner' since $held_iso (fresh)" >&2; exit 75
|
||||
fi
|
||||
fi
|
||||
BODY="$(mktemp)"; printf '%s @ %s\n' "$owner" "$(_now_iso)" > "$BODY"
|
||||
_curl PUT "$(_vault_url "$lockpath")" -H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "lock"; echo "ok: locked by $owner" ;;
|
||||
|
||||
unlock)
|
||||
[ $# -ge 1 ] || usage
|
||||
owner="$1"; lockpath="_agent/locks/vault.lock"
|
||||
_curl GET "$(_vault_url "$lockpath")"
|
||||
if [ "$HTTP" = "200" ]; then
|
||||
cur="$(cat "$RESP")"; held_owner="${cur%% @ *}"
|
||||
[ "$held_owner" = "$owner" ] || { echo "lock owned by '$held_owner', not '$owner' — not releasing" >&2; exit 75; }
|
||||
fi
|
||||
_curl DELETE "$(_vault_url "$lockpath")"
|
||||
[ "$HTTP" = "404" ] || _check "unlock"
|
||||
echo "ok: unlocked" ;;
|
||||
|
||||
scope)
|
||||
# scope show | scope set "<new scope text>"
|
||||
# 'set' archives the prior scope to ## Scope History, replaces ## Scope, and stamps
|
||||
# the scope_updated freshness timestamp — one command instead of three error-prone PATCHes.
|
||||
sub="${1:-show}"; shift || true
|
||||
ccpath="_agent/context/current-context.md"
|
||||
case "$sub" in
|
||||
show)
|
||||
_curl GET "$(_vault_url "$ccpath")"; _check "scope show"
|
||||
cur="$RESP"
|
||||
echo "── Active scope ──"
|
||||
awk '/^## Scope[[:space:]]*$/{f=1;next} /^## /{if(f)exit} f' "$cur"
|
||||
su="$(sed -n 's/^scope_updated:[[:space:]]*//p' "$cur" | head -1)"
|
||||
su="${su//\"/}"
|
||||
echo "scope_updated: ${su:-<missing — drift cannot be detected; run scope set or repair>}"
|
||||
_curl GET "$(_vault_url "_agent/sessions/")"
|
||||
if [ "$HTTP" = "200" ] && [ -n "$su" ]; then
|
||||
n="$(python3 -c "import json,sys;f=json.load(open('$RESP'))['files'];print(sum(1 for x in f if x.endswith('.md') and x[:10]>'$su'))" 2>/dev/null || echo '?')"
|
||||
echo "sessions logged since: ${n}"
|
||||
fi ;;
|
||||
set)
|
||||
[ $# -ge 1 ] || die "scope set needs the new scope text"
|
||||
new="$1"
|
||||
_curl GET "$(_vault_url "$ccpath")"; _check "scope set(read)"
|
||||
prior="$(awk '/^## Scope[[:space:]]*$/{f=1;next} /^## /{if(f)exit} f' "$RESP" \
|
||||
| tr '\n' ' ' | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' | cut -c1-140)"
|
||||
BODY="$(mktemp)"; printf -- '- %s: %s\n' "$(_today)" "${prior:-(prior scope)}" > "$BODY"
|
||||
_curl PATCH "$(_vault_url "$ccpath")" -H 'Operation: prepend' -H 'Target-Type: heading' \
|
||||
-H 'Target: Current Context::Scope History' -H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "scope set(history)"
|
||||
BODY="$(mktemp)"; printf '%s\n' "$new" > "$BODY"
|
||||
_curl PATCH "$(_vault_url "$ccpath")" -H 'Operation: replace' -H 'Target-Type: heading' \
|
||||
-H 'Target: Current Context::Scope' -H 'Content-Type: text/markdown' --data-binary @"$BODY"
|
||||
_check "scope set(replace)"
|
||||
BODY="$(mktemp)"; printf '"%s"' "$(_today)" > "$BODY"
|
||||
_curl PATCH "$(_vault_url "$ccpath")" -H 'Operation: replace' -H 'Target-Type: frontmatter' \
|
||||
-H 'Target: scope_updated' -H 'Content-Type: application/json' --data-binary @"$BODY"
|
||||
if [ "${HTTP:-000}" -ge 400 ]; then
|
||||
die "scope set: body switched, but scope_updated frontmatter is missing (run bootstrap.sh repair to add it) [HTTP $HTTP]"
|
||||
fi
|
||||
echo "ok: scope switched (prior archived to Scope History; scope_updated=$(_today))" ;;
|
||||
*) die "scope: use 'show' or 'set \"<text>\"'" ;;
|
||||
esac ;;
|
||||
|
||||
""|-h|--help|help) usage ;;
|
||||
*) die "unknown command '$cmd' (try: get map ls search put post append patch fm bump delete lock unlock scope)" ;;
|
||||
esac
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
"""echo_index.py — the ECHO entity index.
|
||||
|
||||
A small machine-readable registry at `_agent/index/entities.json` that turns
|
||||
routing/resolve into an O(1) lookup (no fuzzy search per write) and supplies the
|
||||
name->path map used for cross-linking and recall.
|
||||
|
||||
{
|
||||
"schema": 1,
|
||||
"updated": "YYYY-MM-DD",
|
||||
"entities": {
|
||||
"<slug>": {"path": "...", "kind": "...", "title": "...",
|
||||
"aliases": [...], "last_seen": "YYYY-MM-DD"}
|
||||
}
|
||||
}
|
||||
|
||||
Imported by echo_ops.py, sweep.py, and vault_lint.py. Network goes through echo.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import echo # noqa: E402
|
||||
|
||||
INDEX_PATH = "_agent/index/entities.json"
|
||||
|
||||
# kind -> destination folder (the canonical home for that entity kind)
|
||||
KIND_FOLDER = {
|
||||
"person": "resources/people",
|
||||
"company": "resources/companies",
|
||||
"concept": "resources/concepts",
|
||||
"reference": "resources/references",
|
||||
"meeting": "resources/meetings",
|
||||
"project": "projects/active", # new projects default to active
|
||||
"area": "areas", # needs a domain segment
|
||||
"semantic": "_agent/memory/semantic",
|
||||
"episodic": "_agent/memory/episodic",
|
||||
"working": "_agent/memory/working",
|
||||
"skill": "_agent/skills/active",
|
||||
"decision": "decisions/by-date",
|
||||
}
|
||||
# kind -> frontmatter `type:` value
|
||||
KIND_TYPE = {
|
||||
"person": "person", "company": "company", "concept": "concept",
|
||||
"reference": "reference", "meeting": "meeting", "project": "project",
|
||||
"area": "area", "semantic": "semantic-memory", "episodic": "episodic-memory",
|
||||
"working": "working-memory", "skill": "skill", "decision": "decision",
|
||||
}
|
||||
DATE_KINDS = {"meeting", "decision"} # filename carries a YYYY-MM-DD prefix
|
||||
AREA_DOMAINS = ("business", "personal", "learning", "systems")
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
text = str(text).strip().lower()
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
|
||||
return text[:40] or "untitled"
|
||||
|
||||
|
||||
def derive_path(kind: str, slug: str, date: str | None = None, domain: str = "business") -> str:
|
||||
folder = KIND_FOLDER.get(kind)
|
||||
if folder is None:
|
||||
raise echo.EchoError(f"unknown kind '{kind}' (one of: {', '.join(sorted(KIND_FOLDER))})", 2)
|
||||
if kind == "area":
|
||||
if domain not in AREA_DOMAINS:
|
||||
raise echo.EchoError(f"area domain must be one of {AREA_DOMAINS}", 2)
|
||||
return f"areas/{domain}/{slug}.md"
|
||||
if kind in DATE_KINDS:
|
||||
return f"{folder}/{date or echo.today()}-{slug}.md"
|
||||
return f"{folder}/{slug}.md"
|
||||
|
||||
|
||||
_KIND_RULES = [
|
||||
(r"^resources/people/", "person"),
|
||||
(r"^resources/companies/", "company"),
|
||||
(r"^resources/concepts/", "concept"),
|
||||
(r"^resources/references/", "reference"),
|
||||
(r"^resources/meetings/", "meeting"),
|
||||
(r"^projects/(active|incubating|on-hold|archived)/", "project"),
|
||||
(r"^areas/[^/]+/", "area"),
|
||||
(r"^decisions/by-date/", "decision"),
|
||||
(r"^_agent/memory/semantic/", "semantic"),
|
||||
(r"^_agent/memory/episodic/", "episodic"),
|
||||
(r"^_agent/skills/(active|archived)/", "skill"),
|
||||
]
|
||||
|
||||
|
||||
def kind_for_path(path: str) -> str | None:
|
||||
"""The entity kind a vault path belongs to (the inverse of derive_path), or None
|
||||
if the path is not an indexable entity note (journal, sessions, inbox, ...)."""
|
||||
for rx, kind in _KIND_RULES:
|
||||
if re.match(rx, path):
|
||||
return kind
|
||||
return None
|
||||
|
||||
|
||||
def empty_index() -> dict:
|
||||
return {"schema": 1, "updated": echo.today(), "entities": {}}
|
||||
|
||||
|
||||
def load() -> dict:
|
||||
status, body = echo.request("GET", echo.vault_url(INDEX_PATH))
|
||||
if status == 404:
|
||||
return empty_index()
|
||||
echo.check(status, body, "index load")
|
||||
try:
|
||||
d = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return empty_index()
|
||||
d.setdefault("entities", {})
|
||||
return d
|
||||
|
||||
|
||||
def save(index: dict) -> None:
|
||||
index["updated"] = echo.today()
|
||||
body = json.dumps(index, indent=2, ensure_ascii=False).encode("utf-8")
|
||||
status, b = echo.request("PUT", echo.vault_url(INDEX_PATH), data=body,
|
||||
headers={"Content-Type": "application/json"})
|
||||
echo.check(status, b, "index save")
|
||||
|
||||
|
||||
def _norm(s) -> str:
|
||||
return slugify(s)
|
||||
|
||||
|
||||
def resolve(index: dict, mention: str):
|
||||
"""Return (slug, entry) for a mention matched by slug, title, or alias — else (None, None)."""
|
||||
key = _norm(mention)
|
||||
ents = index.get("entities", {})
|
||||
if key in ents:
|
||||
return key, ents[key]
|
||||
plain = str(mention).strip().lower()
|
||||
for slug, e in ents.items():
|
||||
cands = {slug, _norm(e.get("title", "")), str(e.get("title", "")).strip().lower()}
|
||||
cands |= {_norm(a) for a in e.get("aliases", [])}
|
||||
cands |= {str(a).strip().lower() for a in e.get("aliases", [])}
|
||||
if key in cands or plain in cands:
|
||||
return slug, e
|
||||
return None, None
|
||||
|
||||
|
||||
def upsert(index: dict, slug: str, path: str, kind: str, title: str | None = None,
|
||||
aliases=None) -> dict:
|
||||
ents = index.setdefault("entities", {})
|
||||
e = ents.get(slug, {})
|
||||
e["path"] = path
|
||||
e["kind"] = kind
|
||||
e["title"] = title or e.get("title") or slug
|
||||
merged = set(e.get("aliases", [])) | set(aliases or [])
|
||||
e["aliases"] = sorted(a for a in merged if a)
|
||||
e["last_seen"] = echo.today()
|
||||
ents[slug] = e
|
||||
return e
|
||||
|
||||
|
||||
def name_map(index: dict) -> dict:
|
||||
"""Map every resolvable name -> path: slug, normalized title, basename, aliases.
|
||||
Used to resolve `[[wikilinks]]` and entity mentions back to vault paths."""
|
||||
m: dict[str, str] = {}
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
path = e.get("path")
|
||||
if not path:
|
||||
continue
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
base = base[:-3] if base.endswith(".md") else base
|
||||
keys = {slug, _norm(e.get("title", "")), _norm(base)}
|
||||
keys |= {_norm(a) for a in e.get("aliases", [])}
|
||||
for k in keys:
|
||||
if k:
|
||||
m.setdefault(k, path)
|
||||
return m
|
||||
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""echo_links.py — cross-link primitives.
|
||||
|
||||
Parse and add `## Related` wikilinks, and create **bidirectional** links between
|
||||
notes (read-modify-write, idempotent). Links use path-style targets without the
|
||||
`.md` extension (e.g. `[[resources/people/bob-smith]]`), which are unambiguous
|
||||
and resolve in Obsidian. Network goes through echo.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import echo # noqa: E402
|
||||
|
||||
WIKILINK = re.compile(r"\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]")
|
||||
|
||||
|
||||
def first_h1(text: str) -> str:
|
||||
for line in text.splitlines():
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()
|
||||
return ""
|
||||
|
||||
|
||||
def all_wikilinks(text: str) -> list[str]:
|
||||
return [m.group(1).strip() for m in WIKILINK.finditer(text)]
|
||||
|
||||
|
||||
def related_targets(text: str) -> list[str]:
|
||||
"""Wikilink targets under the `## Related` heading only."""
|
||||
out, cap = [], False
|
||||
for line in text.splitlines():
|
||||
if line.strip().lower() == "## related":
|
||||
cap = True
|
||||
continue
|
||||
if cap and line.startswith("## "):
|
||||
break
|
||||
if cap:
|
||||
out += [m.group(1).strip() for m in WIKILINK.finditer(line)]
|
||||
return out
|
||||
|
||||
|
||||
def source_notes(text: str) -> list[str]:
|
||||
"""Plain relative paths listed in the frontmatter `source_notes:` field."""
|
||||
if not text.startswith("---"):
|
||||
return []
|
||||
end = text.find("\n---", 3)
|
||||
fm = text[:end] if end != -1 else text
|
||||
m = re.search(r"(?m)^source_notes:\s*\[(.*?)\]", fm)
|
||||
if not m:
|
||||
return []
|
||||
return [p.strip().strip('"').strip("'") for p in m.group(1).split(",") if p.strip()]
|
||||
|
||||
|
||||
def link_token(path: str) -> str:
|
||||
"""Path-style wikilink target without the .md extension."""
|
||||
return path[:-3] if path.endswith(".md") else path
|
||||
|
||||
|
||||
def add_related(text: str, target_path: str):
|
||||
"""Return (new_text, changed): ensure `- [[<target>]]` exists under ## Related."""
|
||||
token = link_token(target_path)
|
||||
if token in related_targets(text):
|
||||
return text, False
|
||||
bullet = f"- [[{token}]]"
|
||||
trailing_nl = text.endswith("\n")
|
||||
lines = text.splitlines()
|
||||
idx = next((i for i, l in enumerate(lines) if l.strip().lower() == "## related"), None)
|
||||
if idx is None:
|
||||
sep = "" if trailing_nl else "\n"
|
||||
return text + f"{sep}\n## Related\n{bullet}\n", True
|
||||
# insert after the section's existing content (before the next heading / EOF)
|
||||
j = idx + 1
|
||||
while j < len(lines) and not lines[j].startswith("## "):
|
||||
j += 1
|
||||
k = j
|
||||
while k > idx + 1 and not lines[k - 1].strip():
|
||||
k -= 1
|
||||
lines.insert(k, bullet)
|
||||
return "\n".join(lines) + ("\n" if trailing_nl else ""), True
|
||||
|
||||
|
||||
def get_text(path: str) -> str | None:
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
echo.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def put_text(path: str, text: str) -> None:
|
||||
status, body = echo.request("PUT", echo.vault_url(path), data=text.encode("utf-8"),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
echo.check(status, body, f"put {path}")
|
||||
|
||||
|
||||
def add_one(path: str, target_path: str) -> bool:
|
||||
"""Add [[target]] under path's ## Related. Returns True if the file changed."""
|
||||
text = get_text(path)
|
||||
if text is None:
|
||||
return False
|
||||
new, changed = add_related(text, target_path)
|
||||
if changed:
|
||||
put_text(path, new)
|
||||
return changed
|
||||
|
||||
|
||||
def link_bidirectional(a_path: str, b_path: str):
|
||||
"""Add A<->B reciprocal links. Returns (a_changed, b_changed)."""
|
||||
if a_path == b_path:
|
||||
return (False, False)
|
||||
return (add_one(a_path, b_path), add_one(b_path, a_path))
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
"""echo_ops.py — high-level ECHO operations layered on the client + index + links.
|
||||
|
||||
resolve — map a mention to its canonical note path (or where to create it)
|
||||
recall — search, then expand one hop along links/source_notes for connected context
|
||||
link — create bidirectional `## Related` links between two notes
|
||||
capture — one-call write: route + canonical frontmatter + index + auto-link + agent-log
|
||||
|
||||
Network goes through echo.py; routing/aliases through echo_index; links through echo_links.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import echo # noqa: E402
|
||||
import echo_index as idx_mod # noqa: E402
|
||||
import echo_links as links # noqa: E402
|
||||
|
||||
# Existing-entity capture appends a dated bullet under this per-kind heading.
|
||||
LOG_HEADING = {
|
||||
"project": "Session History", "person": "Log", "company": "Log",
|
||||
"semantic": "Observations", "area": "Log", "concept": "Notes",
|
||||
"reference": "Notes", "meeting": "Notes", "decision": "Notes",
|
||||
"skill": "Notes", "episodic": "Notes", "working": "Notes",
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- resolve -----
|
||||
def resolve(mention: str) -> int:
|
||||
index = idx_mod.load()
|
||||
slug, e = idx_mod.resolve(index, mention)
|
||||
if e:
|
||||
print(json.dumps({"match": True, "slug": slug, **e}, ensure_ascii=False, indent=2))
|
||||
else:
|
||||
print(json.dumps({"match": False, "mention": mention,
|
||||
"suggest_slug": idx_mod.slugify(mention),
|
||||
"note": "no index entry — derive a path with the right --kind and create it"},
|
||||
ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- link -----
|
||||
def link(a_path: str, b_path: str) -> int:
|
||||
a_changed, b_changed = links.link_bidirectional(a_path, b_path)
|
||||
print(f"ok: linked {a_path} <-> {b_path} "
|
||||
f"(added: {'A' if a_changed else '-'}{'B' if b_changed else '-'})")
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- recall -----
|
||||
def _resolve_link_to_path(target: str, nmap: dict) -> str | None:
|
||||
target = target.strip()
|
||||
if "/" in target:
|
||||
return target if target.endswith(".md") else target + ".md"
|
||||
return nmap.get(idx_mod.slugify(target))
|
||||
|
||||
|
||||
def _brief(path: str) -> None:
|
||||
text = links.get_text(path)
|
||||
if text is None:
|
||||
return
|
||||
print(f"\n### {path}")
|
||||
fm_type = re.search(r"(?m)^type:\s*(.+)$", text)
|
||||
if fm_type:
|
||||
print(f"_type: {fm_type.group(1).strip()}_")
|
||||
# Prefer a Status paragraph; else the first handful of non-empty body lines.
|
||||
body = text.split("\n---", 2)[-1] if text.startswith("---") else text
|
||||
status = re.search(r"(?ms)^##\s+Status\s*$(.*?)(?=^##\s|\Z)", body)
|
||||
if status and status.group(1).strip():
|
||||
print(status.group(1).strip()[:400])
|
||||
return
|
||||
shown = 0
|
||||
for line in body.splitlines():
|
||||
if not line.strip() or line.startswith("# "):
|
||||
continue
|
||||
print(line[:200])
|
||||
shown += 1
|
||||
if shown >= 8:
|
||||
break
|
||||
|
||||
|
||||
def recall(query, limit: int = 6) -> int:
|
||||
q = " ".join(query) if isinstance(query, list) else query
|
||||
index = idx_mod.load()
|
||||
nmap = idx_mod.name_map(index)
|
||||
|
||||
status, body = echo.request("POST", f"{echo.BASE}/search/simple/?query={urllib.parse.quote_plus(q)}")
|
||||
echo.check(status, body, "recall search")
|
||||
try:
|
||||
results = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
results = []
|
||||
hits = []
|
||||
for r in results[:limit]:
|
||||
fn = r.get("filename") or r.get("path")
|
||||
if fn and fn not in hits:
|
||||
hits.append(fn)
|
||||
rslug, e = idx_mod.resolve(index, q)
|
||||
if e and e.get("path") and e["path"] not in hits:
|
||||
hits.insert(0, e["path"])
|
||||
|
||||
seen = set(hits)
|
||||
neighbors = [] # (via, path)
|
||||
for p in hits:
|
||||
text = links.get_text(p)
|
||||
if text is None:
|
||||
continue
|
||||
targets = set(links.all_wikilinks(text)) | set(links.source_notes(text))
|
||||
for t in targets:
|
||||
tp = _resolve_link_to_path(t, nmap)
|
||||
if tp and tp not in seen:
|
||||
seen.add(tp)
|
||||
neighbors.append((p, tp))
|
||||
|
||||
print(f"== recall: {q} ==")
|
||||
print(f"\n# Primary hits ({len(hits)})")
|
||||
for p in hits:
|
||||
_brief(p)
|
||||
print(f"\n# Linked context ({len(neighbors)})")
|
||||
for via, p in neighbors:
|
||||
print(f"\n(via {via})")
|
||||
_brief(p)
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------------------------------------- agent log -------
|
||||
def ensure_daily_log(line: str) -> None:
|
||||
"""Resilient: ensure today's daily note + its `## Agent Log` heading exist, then
|
||||
idempotently append the line. Best-effort — never raises into the caller."""
|
||||
try:
|
||||
date = echo.today()
|
||||
path = f"journal/daily/{date}.md"
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 404:
|
||||
ts, tb = echo.request("GET", echo.vault_url("journal/templates/daily-note-template.md"))
|
||||
tmpl = tb.decode(errors="replace") if ts == 200 else f"# {date}\n\n## Agent Log\n\n## Related\n"
|
||||
tmpl = tmpl.replace("{{date:YYYY-MM-DD}}", date).replace("{{DATE}}", date)
|
||||
echo.request("PUT", echo.vault_url(path), data=tmpl.encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
text = tmpl
|
||||
else:
|
||||
text = body.decode(errors="replace")
|
||||
if not re.search(r"(?m)^## Agent Log\s*$", text):
|
||||
echo.request("POST", echo.vault_url(path), data=b"\n\n## Agent Log\n",
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 200 and line in body.decode(errors="replace"):
|
||||
return
|
||||
echo.request("PATCH", echo.vault_url(path),
|
||||
data=echo.normalize_patch_body((line + "\n").encode(), "append", "heading"),
|
||||
headers={"Operation": "append", "Target-Type": "heading",
|
||||
"Target": f"{date}::Agent Log", "Content-Type": "text/markdown"})
|
||||
except Exception as exc:
|
||||
print(f"echo_ops: agent-log skipped ({exc})", file=sys.stderr)
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- capture ---
|
||||
def _build_note(fm_type: str, status_v: str, today_s: str, title: str,
|
||||
body_text: str, sources: list[str]) -> str:
|
||||
src = "[" + ", ".join(json.dumps(s) for s in sources) + "]"
|
||||
fm = ["---", f"type: {fm_type}"]
|
||||
if status_v:
|
||||
fm.append(f"status: {status_v}")
|
||||
fm += [f"created: {today_s}", f"updated: {today_s}", "tags: []",
|
||||
"agent_written: true", f"source_notes: {src}", "---", "", f"# {title}", ""]
|
||||
out = "\n".join(fm)
|
||||
if body_text.strip():
|
||||
out += body_text.strip() + "\n"
|
||||
out += "\n## Related\n"
|
||||
return out
|
||||
|
||||
|
||||
def _append_to_existing(path: str, kind: str, today_s: str, body_text: str) -> None:
|
||||
text = links.get_text(path) or ""
|
||||
h1 = links.first_h1(text) or path.rsplit("/", 1)[-1][:-3]
|
||||
heading = LOG_HEADING.get(kind, "Notes")
|
||||
if not re.search(rf"(?m)^##\s+{re.escape(heading)}\s*$", text):
|
||||
echo.request("POST", echo.vault_url(path), data=f"\n\n## {heading}\n".encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
firstline = body_text.strip().splitlines()[0] if body_text.strip() else "(update)"
|
||||
bullet = f"- {today_s}: {firstline}"
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 200 and bullet in body.decode(errors="replace"):
|
||||
return
|
||||
echo.request("PATCH", echo.vault_url(path),
|
||||
data=echo.normalize_patch_body((bullet + "\n").encode(), "append", "heading"),
|
||||
headers={"Operation": "append", "Target-Type": "heading",
|
||||
"Target": f"{h1}::{heading}", "Content-Type": "text/markdown"})
|
||||
echo.cmd_fm(path, "updated", json.dumps(today_s))
|
||||
|
||||
|
||||
def capture(kind: str | None, title: str, file_arg: str | None, status_v: str = "",
|
||||
aliases=None, sources=None, date: str | None = None, domain: str = "business",
|
||||
inbox: bool = False, no_log: bool = False) -> int:
|
||||
body_text = echo.read_body(file_arg).decode("utf-8", errors="replace")
|
||||
today_s = echo.today()
|
||||
aliases = [a.strip() for a in (aliases or []) if a.strip()]
|
||||
sources = [s.strip() for s in (sources or []) if s.strip()]
|
||||
|
||||
# Unknown home -> defer to the inbox (a single idempotent capture line).
|
||||
if inbox or not kind:
|
||||
line = f"- {today_s}: {title}"
|
||||
if body_text.strip():
|
||||
line += f" — {body_text.strip().splitlines()[0]}"
|
||||
return echo.cmd_append("inbox/captures/inbox.md", line)
|
||||
|
||||
slug = idx_mod.slugify(title)
|
||||
index = idx_mod.load()
|
||||
_, existing = idx_mod.resolve(index, title)
|
||||
|
||||
if existing and echo.request("GET", echo.vault_url(existing["path"]))[0] == 200:
|
||||
path = existing["path"]
|
||||
kind = existing.get("kind", kind)
|
||||
_append_to_existing(path, kind, today_s, body_text)
|
||||
action = "updated"
|
||||
else:
|
||||
if not status_v and kind == "project":
|
||||
status_v = "active"
|
||||
path = idx_mod.derive_path(kind, slug, date=date, domain=domain)
|
||||
note = _build_note(idx_mod.KIND_TYPE.get(kind, kind), status_v, today_s, title, body_text, sources)
|
||||
echo.cmd_put(path, echo.temp_file(note.encode()))
|
||||
action = "created"
|
||||
|
||||
idx_mod.upsert(index, slug, path, kind, title, aliases)
|
||||
idx_mod.save(index)
|
||||
|
||||
# auto-link: any other known entity whose name/alias appears in the body, matched
|
||||
# on word boundaries (and >=3 chars) so a short alias can't match inside a word.
|
||||
linked = 0
|
||||
for s2, e2 in index.get("entities", {}).items():
|
||||
if e2.get("path") in (None, path):
|
||||
continue
|
||||
names = [n for n in [e2.get("title", ""), s2, *e2.get("aliases", [])] if len(n) >= 3]
|
||||
if any(re.search(rf"\b{re.escape(n)}\b", body_text, re.IGNORECASE) for n in names):
|
||||
ca, cb = links.link_bidirectional(path, e2["path"])
|
||||
linked += int(ca or cb)
|
||||
|
||||
if not no_log:
|
||||
ensure_daily_log(f"- {today_s}: {action} {kind} [[{links.link_token(path)}]]")
|
||||
|
||||
print(f"ok: {action} {kind} -> {path}" + (f"; auto-linked {linked}" if linked else ""))
|
||||
return 0
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""migrate.py — bring an existing ECHO vault up to the plugin's current schema.
|
||||
|
||||
Reads the marker's schema_version and applies each intervening migration in order.
|
||||
Migrations are idempotent and additive; every destructive step (DELETE/move) is
|
||||
gated behind --apply AND printed first. Default mode is a DRY-RUN plan.
|
||||
Cross-platform: pure Python via echo.py.
|
||||
|
||||
Usage:
|
||||
migrate.py # print the migration plan (no changes)
|
||||
migrate.py --apply # perform the migration (moves/deletes included)
|
||||
|
||||
Env: ECHO_BASE, ECHO_KEY (via echo.py).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import echo # noqa: E402
|
||||
|
||||
CURRENT_SCHEMA = 3
|
||||
|
||||
|
||||
def get_text(path: str) -> str | None:
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
echo.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def list_files(path: str) -> list[str]:
|
||||
if not path.endswith("/"):
|
||||
path += "/"
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 404:
|
||||
return []
|
||||
echo.check(status, body, f"ls {path}")
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return [entry for entry in payload.get("files", []) if not entry.endswith("/")]
|
||||
|
||||
|
||||
def put_text(path: str, text: str) -> None:
|
||||
status, body = echo.request("PUT", echo.vault_url(path), data=text.encode(),
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
echo.check(status, body, f"put {path}")
|
||||
|
||||
|
||||
def delete(path: str) -> None:
|
||||
status, body = echo.request("DELETE", echo.vault_url(path))
|
||||
if status != 404:
|
||||
echo.check(status, body, f"delete {path}")
|
||||
|
||||
|
||||
def move(src: str, dst: str) -> None:
|
||||
text = get_text(src)
|
||||
if text is None:
|
||||
return
|
||||
put_text(dst, text)
|
||||
delete(src)
|
||||
|
||||
|
||||
def do_or_show(apply: bool, desc: str, func=None) -> None:
|
||||
if apply:
|
||||
print(f"migrate: APPLY {desc}")
|
||||
if func:
|
||||
func()
|
||||
else:
|
||||
print(f"migrate: PLAN {desc}")
|
||||
|
||||
|
||||
def marker_schema() -> int:
|
||||
marker = get_text("_agent/echo-vault.md")
|
||||
if marker is None:
|
||||
print("migrate: marker missing — vault not bootstrapped. Run bootstrap.py, not migrate.py.")
|
||||
raise SystemExit(3)
|
||||
for line in marker.splitlines():
|
||||
if line.startswith("schema_version:"):
|
||||
try:
|
||||
return int(line.split(":", 1)[1].strip())
|
||||
except ValueError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Migrate ECHO vault schema")
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
start = marker_schema()
|
||||
print(f"migrate: vault schema_version={start}, plugin schema={CURRENT_SCHEMA} "
|
||||
f"{'(APPLY)' if args.apply else '(dry-run)'}")
|
||||
if start >= CURRENT_SCHEMA:
|
||||
print("migrate: up to date — nothing to do.")
|
||||
return 0
|
||||
|
||||
if start < 1:
|
||||
print("migrate: [0->1] retire in-vault control docs (CLAUDE/BOOTSTRAP/STRUCTURE/index.md)")
|
||||
for filename in ["CLAUDE.md", "BOOTSTRAP.md", "STRUCTURE.md", "index.md"]:
|
||||
if get_text(filename) is not None:
|
||||
do_or_show(args.apply, f"delete vault/{filename} (back it up outside the vault first)",
|
||||
lambda f=filename: delete(f))
|
||||
print("migrate: [0->1] reminder: scrub dangling control-doc [[links]] from Related sections.")
|
||||
|
||||
if start < 2:
|
||||
print("migrate: [1->2] fold reviews/ into journal/ and _agent/health/")
|
||||
for filename in list_files("reviews/weekly"):
|
||||
if filename.endswith(".md"):
|
||||
dst = f"journal/weekly/{filename.replace('-review.md', '.md')}"
|
||||
do_or_show(args.apply, f"move reviews/weekly/{filename} -> {dst}",
|
||||
lambda f=filename, d=dst: move(f"reviews/weekly/{f}", d))
|
||||
for filename in list_files("reviews/monthly"):
|
||||
if filename.endswith(".md"):
|
||||
dst = f"_agent/health/{filename}" if filename.endswith("vault-health.md") else f"journal/monthly/{filename}"
|
||||
do_or_show(args.apply, f"move reviews/monthly/{filename} -> {dst}",
|
||||
lambda f=filename, d=dst: move(f"reviews/monthly/{f}", d))
|
||||
for period in ["quarterly", "annual"]:
|
||||
for filename in list_files(f"reviews/{period}"):
|
||||
if filename.endswith(".md"):
|
||||
dst = f"journal/{period}/{filename}"
|
||||
do_or_show(args.apply, f"move reviews/{period}/{filename} -> {dst}",
|
||||
lambda p=period, f=filename, d=dst: move(f"reviews/{p}/{f}", d))
|
||||
print("migrate: [1->2] reminder: update inbound [[reviews/...]] wikilinks manually.")
|
||||
|
||||
if start < 3:
|
||||
print("migrate: [2->3] add the entity-index folder (cross-link / recall feature)")
|
||||
if get_text("_agent/index/README.md") is None:
|
||||
do_or_show(args.apply, "create _agent/index/README.md",
|
||||
lambda: put_text("_agent/index/README.md",
|
||||
"# index\n\nMachine-maintained entity index. See the echo-memory plugin.\n"))
|
||||
print("migrate: [2->3] then run `sweep.py --apply` to build entities.json and symmetrize links.")
|
||||
|
||||
do_or_show(args.apply, f"set _agent/echo-vault.md schema_version -> {CURRENT_SCHEMA}",
|
||||
lambda: echo.cmd_fm("_agent/echo-vault.md", "schema_version", str(CURRENT_SCHEMA)))
|
||||
if args.apply:
|
||||
print(f"migrate: migration complete -> schema {CURRENT_SCHEMA}. Run sweep.py --apply, then vault_lint.py.")
|
||||
else:
|
||||
print("migrate: dry-run only. Re-run with --apply to perform the moves/deletes above.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# migrate.sh — bring an existing ECHO vault up to the plugin's current schema.
|
||||
#
|
||||
# Reads the marker's schema_version and applies each intervening migration in order.
|
||||
# Migrations are idempotent and additive; every destructive step (DELETE) is gated
|
||||
# behind --apply AND prints what it will do first. Default mode is a DRY-RUN plan.
|
||||
#
|
||||
# Usage:
|
||||
# migrate.sh # print the migration plan (no changes)
|
||||
# migrate.sh --apply # perform the migration (moves/deletes included)
|
||||
#
|
||||
# Env: ECHO_BASE, ECHO_KEY (via echo.sh).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ECHO="$SCRIPT_DIR/echo.sh"
|
||||
CURRENT_SCHEMA=2
|
||||
APPLY=0
|
||||
[ "${1:-}" = "--apply" ] && APPLY=1
|
||||
|
||||
[ -x "$ECHO" ] || chmod +x "$ECHO" 2>/dev/null || true
|
||||
say() { echo "migrate: $*"; }
|
||||
do_or_show() { # do_or_show "<human description>" cmd args...
|
||||
local desc="$1"; shift
|
||||
if [ "$APPLY" = "1" ]; then say "APPLY $desc"; "$@"; else say "PLAN $desc"; fi
|
||||
}
|
||||
|
||||
# ---- Read current schema -----------------------------------------------------
|
||||
if ! marker="$("$ECHO" get _agent/echo-vault.md 2>/dev/null)"; then
|
||||
say "marker missing — vault not bootstrapped. Run bootstrap.sh, not migrate.sh."
|
||||
exit 3
|
||||
fi
|
||||
FROM="$(printf '%s' "$marker" | sed -n 's/^schema_version:[[:space:]]*//p' | head -1)"
|
||||
FROM="${FROM:-0}"
|
||||
say "vault schema_version=$FROM, plugin schema=$CURRENT_SCHEMA $([ "$APPLY" = 1 ] && echo '(APPLY)' || echo '(dry-run)')"
|
||||
|
||||
if [ "$FROM" -ge "$CURRENT_SCHEMA" ] 2>/dev/null; then
|
||||
say "up to date — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ls_files() { "$ECHO" ls "$1" 2>/dev/null | python3 -c 'import sys,json;print("\n".join(json.load(sys.stdin).get("files",[])))' 2>/dev/null || true; }
|
||||
move() { # move SRC DST preserving content (PUT dst <- get src, then delete src)
|
||||
local src="$1" dst="$2"
|
||||
"$ECHO" get "$src" 2>/dev/null | "$ECHO" put "$dst" - >/dev/null
|
||||
"$ECHO" delete "$src" >/dev/null
|
||||
}
|
||||
|
||||
# ---- 0 -> 1 : control docs moved into the plugin -----------------------------
|
||||
mig_0_1() {
|
||||
say "[0->1] retire in-vault control docs (CLAUDE/BOOTSTRAP/STRUCTURE/index.md)"
|
||||
for f in CLAUDE.md BOOTSTRAP.md STRUCTURE.md index.md; do
|
||||
if ECHO_VERIFY=0 "$ECHO" get "$f" >/dev/null 2>&1; then
|
||||
do_or_show "delete vault/$f (back it up outside the vault first)" "$ECHO" delete "$f"
|
||||
fi
|
||||
done
|
||||
say "[0->1] reminder: scrub dangling [[CLAUDE]]/[[BOOTSTRAP]]/[[STRUCTURE]]/[[index]] links from ## Related sections (manual/agent step)."
|
||||
}
|
||||
|
||||
# ---- 1 -> 2 : reviews/ folded into journal/ + _agent/health/ -----------------
|
||||
mig_1_2() {
|
||||
say "[1->2] fold reviews/ into journal/ and _agent/health/"
|
||||
for f in $(ls_files reviews/weekly); do
|
||||
[ "${f%.md}" != "$f" ] || continue
|
||||
dst="journal/weekly/$(printf '%s' "$f" | sed 's/-review\.md$/.md/')"
|
||||
do_or_show "move reviews/weekly/$f -> $dst" move "reviews/weekly/$f" "$dst"
|
||||
done
|
||||
for f in $(ls_files reviews/monthly); do
|
||||
[ "${f%.md}" != "$f" ] || continue
|
||||
case "$f" in
|
||||
*vault-health.md) dst="_agent/health/$f" ;;
|
||||
*) dst="journal/monthly/$f" ;;
|
||||
esac
|
||||
do_or_show "move reviews/monthly/$f -> $dst" move "reviews/monthly/$f" "$dst"
|
||||
done
|
||||
for period in quarterly annual; do
|
||||
for f in $(ls_files "reviews/$period"); do
|
||||
[ "${f%.md}" != "$f" ] || continue
|
||||
do_or_show "move reviews/$period/$f -> journal/$period/$f" move "reviews/$period/$f" "journal/$period/$f"
|
||||
done
|
||||
done
|
||||
say "[1->2] reminder: update inbound [[reviews/...]] wikilinks in ## Related sections (manual/agent step)."
|
||||
}
|
||||
|
||||
[ "$FROM" -lt 1 ] && mig_0_1
|
||||
[ "$FROM" -lt 2 ] && mig_1_2
|
||||
|
||||
# ---- Stamp the marker --------------------------------------------------------
|
||||
do_or_show "set _agent/echo-vault.md schema_version -> $CURRENT_SCHEMA" \
|
||||
"$ECHO" fm _agent/echo-vault.md schema_version "$CURRENT_SCHEMA"
|
||||
|
||||
if [ "$APPLY" = "1" ]; then
|
||||
say "migration complete -> schema $CURRENT_SCHEMA. Run vault-lint.sh to confirm invariants."
|
||||
else
|
||||
say "dry-run only. Re-run with --apply to perform the moves/deletes above."
|
||||
fi
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"$comment": "CANONICAL machine-readable routing manifest for the ECHO vault. This is the single source of truth for 'what paths may be written to'. The human-readable tables in SKILL.md, references/routing-map.md, and references/api-reference.md are DERIVED views of this file — when they disagree, this file wins. vault-lint.sh consumes it to enforce the core rule: if a path matches no route here (and is not a retired path), nothing should be written to it. Patterns are Python regexes matched against vault-root-relative paths (no leading slash, no /vault/ prefix).",
|
||||
"$comment": "CANONICAL machine-readable routing manifest for the ECHO vault. This is the single source of truth for 'what paths may be written to'. The human-readable tables in SKILL.md, references/routing-map.md, and references/api-reference.md are DERIVED views of this file — when they disagree, this file wins. vault_lint.py consumes it to enforce the core rule (if a vault path matches no route here, and is not retired, nothing should be written to it); check_routing.py consumes it to verify the human routing docs stay in sync with this manifest. Patterns are Python regexes matched against vault-root-relative paths (no leading slash, no /vault/ prefix).",
|
||||
"schema_version": 2,
|
||||
"routes": [
|
||||
{ "id": "inbox-captures", "pattern": "^inbox/captures/inbox\\.md$", "method": "POST", "trigger": "Destination unknown at capture time", "distinct_because": "Only path whose contract is deferred routing" },
|
||||
@@ -42,6 +42,7 @@
|
||||
{ "id": "agent-skills-active", "pattern": "^_agent/skills/active/[^/]+\\.md$", "method": "PUT", "trigger": "A skill/plugin catalogued as a capability", "distinct_because": "Catalogs a capability vs the build effort" },
|
||||
{ "id": "agent-skills-archived","pattern": "^_agent/skills/archived/[^/]+\\.md$", "method": "PUT", "trigger": "A catalogued skill is retired", "distinct_because": "Terminal state of the skill catalog" },
|
||||
{ "id": "agent-locks", "pattern": "^_agent/locks/[^/]+\\.lock$", "method": "PUT", "trigger": "Advisory multi-writer lock acquire/release", "distinct_because": "Concurrency coordination, not memory" },
|
||||
{ "id": "agent-index", "pattern": "^_agent/index/[^/]+\\.json$", "method": "PUT", "trigger": "Entity index rebuilt on write/sweep", "distinct_because": "Machine-maintained routing/recall registry, not a note" },
|
||||
|
||||
{ "id": "leaf-readme", "pattern": "^(.+/)?README\\.md$", "method": "PUT", "trigger": "Bootstrap leaf signpost / vault root README", "distinct_because": "Human signpost, not read for routing" }
|
||||
],
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sweep.py — bring an existing ECHO vault up to the current feature spec.
|
||||
|
||||
After upgrading the plugin (entity index, cross-linking, recall), run this once to
|
||||
backfill what older vaults don't have yet:
|
||||
|
||||
1. ensure the `_agent/index/` folder exists,
|
||||
2. **(re)build the entity index** (`_agent/index/entities.json`) from the notes
|
||||
already in the vault (people, companies, projects, concepts, references,
|
||||
meetings, areas, decisions, semantic/episodic memory, skills),
|
||||
3. **symmetrize cross-links** — for every `## Related` link A -> B, ensure the
|
||||
reciprocal B -> A exists (it only adds the missing direction; it never invents
|
||||
new links from body text), and
|
||||
4. stamp the marker `schema_version` to the current schema (3).
|
||||
|
||||
Dry-run by default; pass --apply to write. READ-ONLY without --apply.
|
||||
Cross-platform: pure Python via echo.py.
|
||||
|
||||
Usage: sweep.py [--apply]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import echo # noqa: E402
|
||||
import echo_index as idx_mod # noqa: E402
|
||||
import echo_links as links # noqa: E402
|
||||
|
||||
CURRENT_SCHEMA = 3
|
||||
TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
SKIP_BASENAMES = {"README.md"}
|
||||
|
||||
|
||||
kind_for = idx_mod.kind_for_path
|
||||
|
||||
|
||||
def get(path: str) -> str | None:
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
echo.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def list_dir(path: str):
|
||||
p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/")
|
||||
body = get(p)
|
||||
if body is None:
|
||||
return [], []
|
||||
try:
|
||||
j = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [], []
|
||||
entries = list(j.get("files", [])) + list(j.get("folders", []))
|
||||
files = [e for e in entries if not e.endswith("/")]
|
||||
folders = [e[:-1] for e in entries if e.endswith("/")]
|
||||
return files, folders
|
||||
|
||||
|
||||
def walk(prefix: str = ""):
|
||||
files, folders = list_dir(prefix)
|
||||
for f in files:
|
||||
yield prefix + f
|
||||
for d in folders:
|
||||
yield from walk(f"{prefix}{d}/")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(description="Bring an ECHO vault up to the current feature spec")
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
args = ap.parse_args(argv)
|
||||
apply = args.apply
|
||||
tag = "APPLY" if apply else "PLAN "
|
||||
|
||||
if get("_agent/echo-vault.md") is None:
|
||||
print("sweep: marker missing — vault not bootstrapped. Run bootstrap.py first.", file=sys.stderr)
|
||||
return 3
|
||||
|
||||
print(f"sweep: {'applying changes' if apply else 'dry-run (no writes)'}\n")
|
||||
|
||||
# ---- 1. index folder ------------------------------------------------------
|
||||
if get("_agent/index/README.md") is None:
|
||||
print(f"sweep: {tag} create _agent/index/README.md")
|
||||
if apply:
|
||||
echo.request("PUT", echo.vault_url("_agent/index/README.md"),
|
||||
data=b"# index\n\nMachine-maintained entity index. See the echo-memory plugin.\n",
|
||||
headers={"Content-Type": "text/markdown"})
|
||||
|
||||
all_files = [p for p in walk()
|
||||
if p.endswith(".md") and p.rsplit("/", 1)[-1] not in SKIP_BASENAMES
|
||||
and not TEMPLATE_RE.search(p)]
|
||||
|
||||
# ---- 2. (re)build entity index -------------------------------------------
|
||||
index = idx_mod.load()
|
||||
existing = dict(index.get("entities", {}))
|
||||
rebuilt = idx_mod.empty_index()
|
||||
added = updated = 0
|
||||
for path in all_files:
|
||||
kind = kind_for(path)
|
||||
if kind is None:
|
||||
continue
|
||||
base = path.rsplit("/", 1)[-1][:-3]
|
||||
slug = idx_mod.slugify(base)
|
||||
text = get(path) or ""
|
||||
title = links.first_h1(text) or base
|
||||
prev = existing.get(slug)
|
||||
aliases = prev.get("aliases", []) if prev else []
|
||||
idx_mod.upsert(rebuilt, slug, path, kind, title, aliases)
|
||||
if not prev:
|
||||
added += 1
|
||||
elif prev.get("path") != path or prev.get("title") != title:
|
||||
updated += 1
|
||||
print(f"sweep: {tag} entity index — {len(rebuilt['entities'])} entities "
|
||||
f"({added} new, {updated} changed)")
|
||||
if apply:
|
||||
idx_mod.save(rebuilt)
|
||||
|
||||
# ---- 3. symmetrize existing Related links --------------------------------
|
||||
fileset = set(all_files)
|
||||
basemap: dict[str, str] = {}
|
||||
for p in all_files:
|
||||
basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)
|
||||
|
||||
def resolve_target(t: str) -> str | None:
|
||||
t = t.strip()
|
||||
if "/" in t:
|
||||
cand = t if t.endswith(".md") else t + ".md"
|
||||
return cand if cand in fileset else None
|
||||
return basemap.get(idx_mod.slugify(t))
|
||||
|
||||
missing = set() # (target_path, source_path) — reciprocal link to add on target
|
||||
for path in all_files:
|
||||
text = get(path)
|
||||
if text is None:
|
||||
continue
|
||||
for tgt in links.related_targets(text):
|
||||
tp = resolve_target(tgt)
|
||||
if not tp or tp == path:
|
||||
continue
|
||||
back = {resolve_target(t) for t in links.related_targets(get(tp) or "")}
|
||||
if path not in back:
|
||||
missing.add((tp, path))
|
||||
missing = sorted(missing)
|
||||
print(f"sweep: {tag} reciprocal links — {len(missing)} to add")
|
||||
for tp, sp in missing[:40]:
|
||||
print(f" {tp} += [[{links.link_token(sp)}]]")
|
||||
if len(missing) > 40:
|
||||
print(f" ... and {len(missing) - 40} more")
|
||||
if apply:
|
||||
for tp, sp in missing:
|
||||
links.add_one(tp, sp)
|
||||
|
||||
# ---- 4. stamp schema ------------------------------------------------------
|
||||
marker = get("_agent/echo-vault.md") or ""
|
||||
cur = next((int(m) for m in re.findall(r"(?m)^schema_version:\s*(\d+)", marker)), 0)
|
||||
if cur < CURRENT_SCHEMA:
|
||||
print(f"sweep: {tag} schema_version {cur} -> {CURRENT_SCHEMA}")
|
||||
if apply:
|
||||
echo.cmd_fm("_agent/echo-vault.md", "schema_version", str(CURRENT_SCHEMA))
|
||||
|
||||
print(f"\nsweep: {'done — run vault_lint.py to confirm invariants' if apply else 'dry-run complete. Re-run with --apply to write.'}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline regression tests for the ECHO tooling. No network, no vault.
|
||||
|
||||
Run: python3 test_echo_client.py (or: pytest test_echo_client.py)
|
||||
|
||||
Covers echo.py body/scalar normalization, the routing-view consistency checker's
|
||||
helpers, and — as the guard for improvement #4 — asserts the shipped docs are in
|
||||
sync with routing.json.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
import echo # noqa: E402
|
||||
import check_routing # noqa: E402
|
||||
import echo_index # noqa: E402
|
||||
import echo_links # noqa: E402
|
||||
|
||||
|
||||
def test_heading_replace_body_is_newline_guarded() -> None:
|
||||
assert echo.normalize_patch_body(b"- [x] done\n", "replace", "heading") == b"\n- [x] done\n"
|
||||
|
||||
|
||||
def test_heading_append_body_ends_with_newline() -> None:
|
||||
assert echo.normalize_patch_body(b"- item", "append", "heading") == b"- item\n"
|
||||
|
||||
|
||||
def test_frontmatter_plain_string_becomes_json_scalar() -> None:
|
||||
assert json.loads(echo.normalize_json_scalar("Fabrication Lead")) == "Fabrication Lead"
|
||||
|
||||
|
||||
def test_frontmatter_existing_json_is_preserved() -> None:
|
||||
assert echo.normalize_json_scalar('"2026-06-20"') == b'"2026-06-20"'
|
||||
|
||||
|
||||
def test_stem_extracts_literal_directory_prefix() -> None:
|
||||
assert check_routing.stem(r"^projects/active/[^/]+\.md$") == "projects/active/"
|
||||
assert check_routing.stem(r"^areas/(business|personal)/[^/]+\.md$") == "areas/"
|
||||
assert check_routing.stem(r"^journal/daily/\d{4}-\d{2}-\d{2}\.md$") == "journal/daily/"
|
||||
|
||||
|
||||
def test_concretize_substitutes_placeholders() -> None:
|
||||
assert check_routing.concretize("projects/<lifecycle>/<slug>.md") == "projects/active/sample.md"
|
||||
assert check_routing.concretize("_agent/sessions/YYYY-MM-DD-HHMM-<slug>.md") == "_agent/sessions/2026-01-02-0900-sample.md"
|
||||
assert check_routing.concretize("journal/weekly/YYYY-Www.md") == "journal/weekly/2026-W01.md"
|
||||
|
||||
|
||||
def test_looks_like_path_filters_non_paths() -> None:
|
||||
assert check_routing.looks_like_path("inbox/captures/inbox.md")
|
||||
assert check_routing.looks_like_path("_agent/locks/vault.lock")
|
||||
assert not check_routing.looks_like_path("application/vnd.olrapi.document-map+json")
|
||||
assert not check_routing.looks_like_path("Operator Preferences::Fact / Pattern")
|
||||
assert not check_routing.looks_like_path("status: active")
|
||||
|
||||
|
||||
def test_docs_are_in_sync_with_routing_json() -> None:
|
||||
# The guard for improvement #4: SKILL.md / routing-map.md / api-reference.md
|
||||
# must stay consistent with routing.json.
|
||||
assert check_routing.main() == 0
|
||||
|
||||
|
||||
def test_index_slugify_and_derive_path() -> None:
|
||||
assert echo_index.slugify("Bob Smith!") == "bob-smith"
|
||||
assert echo_index.derive_path("person", "bob-smith") == "resources/people/bob-smith.md"
|
||||
assert echo_index.derive_path("project", "echo") == "projects/active/echo.md"
|
||||
assert echo_index.derive_path("area", "ops", domain="business") == "areas/business/ops.md"
|
||||
assert echo_index.derive_path("decision", "x", date="2026-01-02") == "decisions/by-date/2026-01-02-x.md"
|
||||
|
||||
|
||||
def test_index_resolve_matches_alias_and_title() -> None:
|
||||
index = {"entities": {"bob-smith": {"path": "resources/people/bob-smith.md",
|
||||
"kind": "person", "title": "Bob Smith", "aliases": ["bob"]}}}
|
||||
assert echo_index.resolve(index, "Bob Smith")[0] == "bob-smith"
|
||||
assert echo_index.resolve(index, "bob")[0] == "bob-smith"
|
||||
assert echo_index.resolve(index, "nobody")[0] is None
|
||||
|
||||
|
||||
def test_kind_for_path() -> None:
|
||||
assert echo_index.kind_for_path("resources/people/x.md") == "person"
|
||||
assert echo_index.kind_for_path("projects/on-hold/x.md") == "project"
|
||||
assert echo_index.kind_for_path("journal/daily/2026-01-01.md") is None
|
||||
|
||||
|
||||
def test_links_add_related_is_idempotent_and_bidirectional_token() -> None:
|
||||
text = "---\ntype: person\n---\n\n# Bob\n\n## Related\n"
|
||||
new, changed = echo_links.add_related(text, "resources/companies/mpm.md")
|
||||
assert changed and "[[resources/companies/mpm]]" in new
|
||||
again, changed2 = echo_links.add_related(new, "resources/companies/mpm.md")
|
||||
assert not changed2 and again == new
|
||||
|
||||
|
||||
def test_links_create_related_section_when_absent() -> None:
|
||||
text = "---\ntype: concept\n---\n\n# Alpha\n\nbody\n"
|
||||
new, changed = echo_links.add_related(text, "resources/concepts/beta.md")
|
||||
assert changed and "## Related" in new and "[[resources/concepts/beta]]" in new
|
||||
|
||||
|
||||
def _run_all() -> int:
|
||||
tests = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
|
||||
failed = 0
|
||||
for test in tests:
|
||||
try:
|
||||
test()
|
||||
print(f"ok {test.__name__}")
|
||||
except AssertionError as exc:
|
||||
failed += 1
|
||||
print(f"FAIL {test.__name__}: {exc}")
|
||||
print(f"\n{len(tests) - failed}/{len(tests)} passed")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(_run_all())
|
||||
@@ -1,318 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# vault-lint.sh — mechanically assert ECHO vault invariants.
|
||||
#
|
||||
# Catches the recurring "invariant violation" bugs that prose rules can't enforce:
|
||||
# folder<->status drift, duplicate slugs, wikilinks leaking into frontmatter,
|
||||
# duplicate "## Agent Log" headings, stale active projects, aging inbox captures,
|
||||
# impossible dates, bad status values, missing frontmatter, broken source_notes, and
|
||||
# paths that no route in routing.json permits. Invoked by the monthly Vault Health
|
||||
# pass (see SKILL.md), but safe to run any time — it is READ-ONLY.
|
||||
#
|
||||
# Exit status: 0 = clean, 1 = violations found, 2 = vault unreachable,
|
||||
# 3 = vault not bootstrapped (marker missing).
|
||||
#
|
||||
# Config (env overrides):
|
||||
# ECHO_BASE (default https://echoapi.alwisp.com)
|
||||
# ECHO_KEY (default the plugin's bearer token)
|
||||
# ECHO_TODAY (default the machine date) — pass the conversation's currentDate so
|
||||
# stale/aging math uses the SAME clock the agent writes with (YYYY-MM-DD)
|
||||
# STALE_DAYS (default 30) INBOX_DAYS (default 14)
|
||||
#
|
||||
# routing.json (canonical route manifest) is read from this script's own directory.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
ECHO_BASE="${ECHO_BASE:-https://echoapi.alwisp.com}"
|
||||
ECHO_KEY="${ECHO_KEY:-241265fbe6830934a9a4ad3e69335f64a42153b663aa5b0017cb1ea1217b2bab}"
|
||||
STALE_DAYS="${STALE_DAYS:-30}"
|
||||
INBOX_DAYS="${INBOX_DAYS:-14}"
|
||||
SCOPE_STALE_SESSIONS="${SCOPE_STALE_SESSIONS:-3}"
|
||||
ECHO_TODAY="${ECHO_TODAY:-$(date +%Y-%m-%d)}"
|
||||
|
||||
ECHO_BASE="$ECHO_BASE" ECHO_KEY="$ECHO_KEY" STALE_DAYS="$STALE_DAYS" INBOX_DAYS="$INBOX_DAYS" \
|
||||
SCOPE_STALE_SESSIONS="$SCOPE_STALE_SESSIONS" \
|
||||
ECHO_TODAY="$ECHO_TODAY" ROUTING_JSON="$SCRIPT_DIR/routing.json" \
|
||||
python3 - <<'PY'
|
||||
import os, sys, json, re, datetime, urllib.request, urllib.error
|
||||
|
||||
BASE = os.environ["ECHO_BASE"].rstrip("/")
|
||||
KEY = os.environ["ECHO_KEY"]
|
||||
STALE_DAYS = int(os.environ["STALE_DAYS"])
|
||||
INBOX_DAYS = int(os.environ["INBOX_DAYS"])
|
||||
SCOPE_STALE_SESSIONS = int(os.environ["SCOPE_STALE_SESSIONS"])
|
||||
TODAY = datetime.date.fromisoformat(os.environ["ECHO_TODAY"])
|
||||
ROUTING_JSON = os.environ["ROUTING_JSON"]
|
||||
LIFECYCLES = ["active", "incubating", "on-hold", "archived"]
|
||||
SKIP = {"README.md", "project-template.md", "decision-template.md"}
|
||||
REQUIRED_FM = ("type", "created")
|
||||
# Project status vocabulary IS enforced (status must equal the lifecycle folder) by the
|
||||
# folder/status check below. Other note kinds (decisions/concepts) carry free-form status
|
||||
# vocab (accepted, shipped, reference, ...), so there is no global status allow-list.
|
||||
|
||||
# optional real YAML parser; fall back to a tolerant line parser
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
HAVE_YAML = True
|
||||
except Exception:
|
||||
HAVE_YAML = False
|
||||
|
||||
violations = []
|
||||
def flag(check, msg): violations.append((check, msg))
|
||||
|
||||
def get(path):
|
||||
"""GET /vault/<path>. Returns text, or None on 404. Raises on hard failure."""
|
||||
req = urllib.request.Request(f"{BASE}/vault/{path}",
|
||||
headers={"Authorization": f"Bearer {KEY}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=20) as r:
|
||||
return r.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 404:
|
||||
return None
|
||||
raise
|
||||
|
||||
def list_dir(path):
|
||||
"""Return (files, folders) for a vault directory. Directories may arrive either in a
|
||||
'folders' key OR as 'files' entries ending in '/'; handle both. Root is '' -> /vault/.
|
||||
Tolerates non-404 errors (e.g. a 400 on an odd path) by returning empty."""
|
||||
p = "" if path in ("", "/") else (path if path.endswith("/") else path + "/")
|
||||
try:
|
||||
body = get(p)
|
||||
except urllib.error.HTTPError:
|
||||
return [], []
|
||||
if body is None:
|
||||
return [], []
|
||||
try:
|
||||
j = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [], []
|
||||
entries = list(j.get("files", [])) + list(j.get("folders", []))
|
||||
files = [e for e in entries if not e.endswith("/")]
|
||||
folders = [e[:-1] for e in entries if e.endswith("/")]
|
||||
return files, folders
|
||||
|
||||
def walk(prefix=""):
|
||||
"""Yield every file path under prefix (recursive). prefix is '' or ends with '/'."""
|
||||
files, folders = list_dir(prefix)
|
||||
for f in files:
|
||||
yield prefix + f
|
||||
for d in folders:
|
||||
yield from walk(f"{prefix}{d}/")
|
||||
|
||||
def split_frontmatter(text):
|
||||
"""Return (raw_yaml_str, body) splitting on anchored ^---$ delimiters. ('', text) if none."""
|
||||
if not text:
|
||||
return "", ""
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return "", text
|
||||
for i in range(1, len(lines)):
|
||||
if lines[i].strip() == "---":
|
||||
return "\n".join(lines[1:i]), "\n".join(lines[i+1:])
|
||||
return "", text # unterminated block -> treat as no frontmatter
|
||||
|
||||
def parse_fm(text):
|
||||
"""Return (raw_yaml_str, dict). Uses PyYAML when available, else a tolerant parser."""
|
||||
raw, _ = split_frontmatter(text)
|
||||
if not raw:
|
||||
return "", {}
|
||||
if HAVE_YAML:
|
||||
try:
|
||||
d = yaml.safe_load(raw)
|
||||
return raw, (d if isinstance(d, dict) else {})
|
||||
except Exception:
|
||||
pass
|
||||
# fallback: scalar + simple inline-list lines (keys may contain digits, _, -)
|
||||
fields = {}
|
||||
for line in raw.splitlines():
|
||||
m = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||||
if m:
|
||||
v = m.group(2).strip()
|
||||
if v.startswith("[") and v.endswith("]"):
|
||||
v = [x.strip().strip('"').strip("'") for x in v[1:-1].split(",") if x.strip()]
|
||||
else:
|
||||
v = v.strip('"').strip("'")
|
||||
fields[m.group(1)] = v
|
||||
return raw, fields
|
||||
|
||||
def parse_date(s):
|
||||
m = re.match(r"(\d{4}-\d{2}-\d{2})", str(s or ""))
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
return datetime.date.fromisoformat(m.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def as_list(v):
|
||||
if v is None or v == "":
|
||||
return []
|
||||
return v if isinstance(v, list) else [v]
|
||||
|
||||
# ---- Reachability + bootstrap probe (M2: do NOT silently report clean) -------
|
||||
try:
|
||||
if get("_agent/echo-vault.md") is None:
|
||||
print("vault-lint: marker missing — vault not bootstrapped (run bootstrap.sh).", file=sys.stderr)
|
||||
sys.exit(3)
|
||||
except Exception as e:
|
||||
print(f"vault-lint: vault unreachable ({e}).", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# ---- Load canonical routing manifest (S3) ------------------------------------
|
||||
ROUTES, RETIRED = [], []
|
||||
try:
|
||||
with open(ROUTING_JSON) as fh:
|
||||
rj = json.load(fh)
|
||||
ROUTES = [(r["id"], re.compile(r["pattern"])) for r in rj.get("routes", [])]
|
||||
RETIRED = [(re.compile(r["pattern"]), r.get("replacement", "")) for r in rj.get("retired", [])]
|
||||
except Exception as e:
|
||||
flag("routing-manifest", f"could not load routing.json ({e}) — path checks skipped")
|
||||
|
||||
# ---- Single full walk feeds every path-level check ---------------------------
|
||||
all_files = list(walk())
|
||||
|
||||
def route_for(path):
|
||||
for rid, rx in ROUTES:
|
||||
if rx.match(path):
|
||||
return rid
|
||||
return None
|
||||
|
||||
# Path membership + retired-path detection (S3)
|
||||
for path in all_files:
|
||||
if ROUTES and route_for(path) is None:
|
||||
hit = next((repl for rx, repl in RETIRED if rx.match(path)), None)
|
||||
if hit is not None:
|
||||
flag("retired-path", f"{path}: retired location — should be {hit}")
|
||||
else:
|
||||
flag("unknown-path", f"{path}: matches no route in routing.json")
|
||||
|
||||
# ---- Per-note frontmatter checks (M5) ----------------------------------------
|
||||
TEMPLATE_RE = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
for path in all_files:
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
if base in SKIP or TEMPLATE_RE.search(path) or not path.endswith(".md"):
|
||||
continue
|
||||
text = get(path)
|
||||
if text is None:
|
||||
continue
|
||||
raw, fm = parse_fm(text)
|
||||
|
||||
# wikilinks anywhere in frontmatter (widened sweep — all folders)
|
||||
if "[[" in raw:
|
||||
flag("frontmatter-wikilink", f"{path}: '[[...]]' inside frontmatter")
|
||||
|
||||
# missing required frontmatter
|
||||
missing = [k for k in REQUIRED_FM if not str(fm.get(k, "")).strip()]
|
||||
if fm and missing:
|
||||
flag("missing-frontmatter", f"{path}: missing {', '.join(missing)}")
|
||||
|
||||
# impossible dates: updated < created
|
||||
c, u = parse_date(fm.get("created")), parse_date(fm.get("updated"))
|
||||
if c and u and u < c:
|
||||
flag("date-order", f"{path}: updated {u} is before created {c}")
|
||||
if u and u > TODAY:
|
||||
flag("future-date", f"{path}: updated {u} is in the future (today {TODAY})")
|
||||
|
||||
# source_notes hygiene: plain relative paths, never wikilinks, no self-reference
|
||||
for sn in as_list(fm.get("source_notes")):
|
||||
s = str(sn)
|
||||
if "[[" in s:
|
||||
flag("source-notes-wikilink", f"{path}: source_notes contains a wikilink '{s}'")
|
||||
|
||||
# ---- Projects: folder<->status, stale active, duplicate slugs ----------------
|
||||
slug_homes = {}
|
||||
for lc in LIFECYCLES:
|
||||
files, _ = list_dir(f"projects/{lc}")
|
||||
for fn in files:
|
||||
if fn.endswith("/") or fn in SKIP or not fn.endswith(".md"):
|
||||
continue
|
||||
slug = fn[:-3]
|
||||
slug_homes.setdefault(slug, []).append(lc)
|
||||
text = get(f"projects/{lc}/{fn}")
|
||||
if text is None:
|
||||
continue
|
||||
_, fm = parse_fm(text)
|
||||
status = str(fm.get("status", "")).strip().strip('"').strip("'")
|
||||
if status and status != lc:
|
||||
flag("folder/status", f"projects/{lc}/{fn}: status='{status}' but folder='{lc}'")
|
||||
if lc == "active":
|
||||
d = parse_date(fm.get("updated"))
|
||||
if d and (TODAY - d).days > STALE_DAYS:
|
||||
flag("stale-active", f"projects/active/{fn}: updated {d} ({(TODAY-d).days}d ago) — consider on-hold/")
|
||||
for slug, homes in slug_homes.items():
|
||||
if len(homes) > 1:
|
||||
flag("duplicate-slug", f"'{slug}' exists in {', '.join(homes)}")
|
||||
|
||||
# ---- Daily notes: duplicate "## Agent Log" headings --------------------------
|
||||
for path in all_files:
|
||||
if not re.match(r"^journal/daily/.*\.md$", path):
|
||||
continue
|
||||
text = get(path) or ""
|
||||
n = len(re.findall(r"(?m)^## Agent Log\s*$", text))
|
||||
if n > 1:
|
||||
flag("duplicate-agent-log", f"{path}: {n} '## Agent Log' headings")
|
||||
|
||||
# ---- Inbox: captures aging past INBOX_DAYS -----------------------------------
|
||||
inbox = get("inbox/captures/inbox.md") or ""
|
||||
for line in inbox.splitlines():
|
||||
m = re.match(r"^\s*-\s*(\d{4}-\d{2}-\d{2})\b", line)
|
||||
if m:
|
||||
d = parse_date(m.group(1))
|
||||
if d and (TODAY - d).days > INBOX_DAYS:
|
||||
flag("aging-inbox", f"inbox capture {d} ({(TODAY-d).days}d): {line.strip()[:80]}")
|
||||
|
||||
# ---- Scope freshness (drift detector) ----------------------------------------
|
||||
# Scope is the most churn-prone state (Jason runs several sessions/day across topics).
|
||||
# It has no natural staleness signal, so drift is otherwise invisible. Rule: if N+ session
|
||||
# logs are dated AFTER current-context's scope_updated, the recorded scope may no longer
|
||||
# reflect current work — surface it for a human glance (advisory, like every health finding).
|
||||
cc = get("_agent/context/current-context.md")
|
||||
if cc is not None:
|
||||
_, ccfm = parse_fm(cc)
|
||||
su = parse_date(ccfm.get("scope_updated"))
|
||||
if su is None:
|
||||
flag("scope-no-timestamp",
|
||||
"_agent/context/current-context.md: no scope_updated frontmatter — scope drift cannot be detected; add it (bootstrap.sh repair) and switch scope via `echo.sh scope set`")
|
||||
else:
|
||||
since = [p for p in all_files
|
||||
if (m := re.match(r"^_agent/sessions/(\d{4}-\d{2}-\d{2})", p))
|
||||
and (d := parse_date(m.group(1))) and d > su]
|
||||
if len(since) >= SCOPE_STALE_SESSIONS:
|
||||
flag("scope-stale",
|
||||
f"scope set {su}; {len(since)} session(s) logged since without a switch — confirm it still reflects current work (or run `echo.sh scope set`)")
|
||||
|
||||
# ---- Report ------------------------------------------------------------------
|
||||
if not violations:
|
||||
print("vault-lint: clean — all invariants hold.")
|
||||
sys.exit(0)
|
||||
|
||||
print(f"vault-lint: {len(violations)} violation(s) found\n")
|
||||
by = {}
|
||||
for check, msg in violations:
|
||||
by.setdefault(check, []).append(msg)
|
||||
labels = {
|
||||
"folder/status": "Folder <-> status mismatch",
|
||||
"duplicate-slug": "Duplicate slug across lifecycle folders",
|
||||
"frontmatter-wikilink": "Wikilink in frontmatter (breaks reading view)",
|
||||
"duplicate-agent-log": "Duplicate '## Agent Log' heading",
|
||||
"stale-active": f"Stale active project (updated > {STALE_DAYS}d)",
|
||||
"aging-inbox": f"Inbox capture aging (> {INBOX_DAYS}d)",
|
||||
"unknown-path": "Path matches no route in routing.json",
|
||||
"retired-path": "Write to a retired/dead path",
|
||||
"missing-frontmatter": "Missing required frontmatter field",
|
||||
"date-order": "updated earlier than created",
|
||||
"future-date": "updated date is in the future",
|
||||
"source-notes-wikilink": "Wikilink in source_notes (must be plain paths)",
|
||||
"routing-manifest": "routing.json problem",
|
||||
"scope-no-timestamp": "current-context has no scope_updated (drift undetectable)",
|
||||
"scope-stale": f"Scope may have drifted (>= {SCOPE_STALE_SESSIONS} sessions since last switch)",
|
||||
}
|
||||
for check, msgs in by.items():
|
||||
print(f"## {labels.get(check, check)}")
|
||||
for m in msgs:
|
||||
print(f" - {m}")
|
||||
print()
|
||||
sys.exit(1)
|
||||
PY
|
||||
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env python3
|
||||
"""vault_lint.py — mechanically assert ECHO vault invariants. READ-ONLY.
|
||||
|
||||
Catches the recurring drift bugs prose rules can't enforce: folder<->status
|
||||
mismatch, duplicate slugs, wikilinks leaking into frontmatter, duplicate
|
||||
"## Agent Log" headings, stale active projects, aging inbox captures, impossible
|
||||
dates, missing frontmatter, broken source_notes, scope drift, and paths that no
|
||||
route in routing.json permits. Cross-platform: pure Python via echo.py.
|
||||
|
||||
Exit status: 0 = clean, 1 = violations found, 2 = vault unreachable,
|
||||
3 = vault not bootstrapped (marker missing).
|
||||
|
||||
Config (env overrides):
|
||||
ECHO_BASE, ECHO_KEY (via echo.py)
|
||||
ECHO_TODAY (default machine date) — pass the conversation's currentDate so
|
||||
stale/aging math uses the same clock the agent writes with.
|
||||
STALE_DAYS (default 30) INBOX_DAYS (default 14) SCOPE_STALE_SESSIONS (default 3)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
import echo # noqa: E402
|
||||
import echo_index as idx_mod # noqa: E402
|
||||
import echo_links as links # noqa: E402
|
||||
|
||||
TODAY = dt.date.fromisoformat(os.environ.get("ECHO_TODAY") or dt.date.today().isoformat())
|
||||
STALE_DAYS = int(os.environ.get("STALE_DAYS", "30"))
|
||||
INBOX_DAYS = int(os.environ.get("INBOX_DAYS", "14"))
|
||||
SCOPE_STALE_SESSIONS = int(os.environ.get("SCOPE_STALE_SESSIONS", "3"))
|
||||
LIFECYCLES = ["active", "incubating", "on-hold", "archived"]
|
||||
SKIP = {"README.md", "project-template.md", "decision-template.md"}
|
||||
REQUIRED_FM = ("type", "created")
|
||||
|
||||
violations: list[tuple[str, str]] = []
|
||||
|
||||
|
||||
def flag(check: str, message: str) -> None:
|
||||
violations.append((check, message))
|
||||
|
||||
|
||||
def get(path: str) -> str | None:
|
||||
status, body = echo.request("GET", echo.vault_url(path))
|
||||
if status == 404:
|
||||
return None
|
||||
echo.check(status, body, f"get {path}")
|
||||
return body.decode(errors="replace")
|
||||
|
||||
|
||||
def list_dir(path: str) -> tuple[list[str], list[str]]:
|
||||
if path and not path.endswith("/"):
|
||||
path += "/"
|
||||
body = get(path)
|
||||
if body is None:
|
||||
return [], []
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except json.JSONDecodeError:
|
||||
return [], []
|
||||
entries = list(payload.get("files", [])) + list(payload.get("folders", []))
|
||||
files = [entry for entry in entries if not entry.endswith("/")]
|
||||
folders = [entry[:-1] for entry in entries if entry.endswith("/")]
|
||||
return files, folders
|
||||
|
||||
|
||||
def walk(prefix: str = ""):
|
||||
files, folders = list_dir(prefix)
|
||||
for filename in files:
|
||||
yield prefix + filename
|
||||
for folder in folders:
|
||||
yield from walk(f"{prefix}{folder}/")
|
||||
|
||||
|
||||
def split_frontmatter(text: str) -> tuple[str, str]:
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return "", text
|
||||
for index in range(1, len(lines)):
|
||||
if lines[index].strip() == "---":
|
||||
return "\n".join(lines[1:index]), "\n".join(lines[index + 1:])
|
||||
return "", text
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[str, dict[str, object]]:
|
||||
raw, _ = split_frontmatter(text)
|
||||
fields: dict[str, object] = {}
|
||||
for line in raw.splitlines():
|
||||
match = re.match(r"^([A-Za-z_][\w-]*):\s*(.*)$", line)
|
||||
if not match:
|
||||
continue
|
||||
value: object = match.group(2).strip().strip('"').strip("'")
|
||||
if isinstance(value, str) and value.startswith("[") and value.endswith("]"):
|
||||
value = [part.strip().strip('"').strip("'") for part in value[1:-1].split(",") if part.strip()]
|
||||
fields[match.group(1)] = value
|
||||
return raw, fields
|
||||
|
||||
|
||||
def parse_date(value: object) -> dt.date | None:
|
||||
match = re.match(r"(\d{4}-\d{2}-\d{2})", str(value or ""))
|
||||
if not match:
|
||||
return None
|
||||
try:
|
||||
return dt.date.fromisoformat(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def as_list(value: object) -> list[object]:
|
||||
if value in (None, ""):
|
||||
return []
|
||||
return value if isinstance(value, list) else [value]
|
||||
|
||||
|
||||
def route_matchers():
|
||||
routing = json.loads((SCRIPT_DIR / "routing.json").read_text(encoding="utf-8"))
|
||||
routes = [(item["id"], re.compile(item["pattern"])) for item in routing.get("routes", [])]
|
||||
retired = [(re.compile(item["pattern"]), item.get("replacement", "")) for item in routing.get("retired", [])]
|
||||
return routes, retired
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
if get("_agent/echo-vault.md") is None:
|
||||
print("vault-lint: marker missing — vault not bootstrapped (run bootstrap.py).", file=sys.stderr)
|
||||
return 3
|
||||
except Exception as exc:
|
||||
print(f"vault-lint: vault unreachable ({exc}).", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
routes, retired = route_matchers()
|
||||
except Exception as exc:
|
||||
routes, retired = [], []
|
||||
flag("routing-manifest", f"could not load routing.json ({exc}) — path checks skipped")
|
||||
|
||||
all_files = list(walk())
|
||||
|
||||
# Path membership + retired-path detection
|
||||
for path in all_files:
|
||||
if routes and not any(rx.match(path) for _, rx in routes):
|
||||
replacement = next((repl for rx, repl in retired if rx.match(path)), None)
|
||||
if replacement is not None:
|
||||
flag("retired-path", f"{path}: retired location — should be {replacement}")
|
||||
else:
|
||||
flag("unknown-path", f"{path}: matches no route in routing.json")
|
||||
|
||||
# Per-note frontmatter checks
|
||||
template_re = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
for path in all_files:
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
if base in SKIP or template_re.search(path) or not path.endswith(".md"):
|
||||
continue
|
||||
text = get(path) or ""
|
||||
raw, fm = parse_frontmatter(text)
|
||||
if "[[" in raw:
|
||||
flag("frontmatter-wikilink", f"{path}: '[[...]]' inside frontmatter")
|
||||
missing = [k for k in REQUIRED_FM if not str(fm.get(k, "")).strip()]
|
||||
if fm and missing:
|
||||
flag("missing-frontmatter", f"{path}: missing {', '.join(missing)}")
|
||||
created, updated = parse_date(fm.get("created")), parse_date(fm.get("updated"))
|
||||
if created and updated and updated < created:
|
||||
flag("date-order", f"{path}: updated {updated} is before created {created}")
|
||||
if updated and updated > TODAY:
|
||||
flag("future-date", f"{path}: updated {updated} is in the future (today {TODAY})")
|
||||
for source_note in as_list(fm.get("source_notes")):
|
||||
if "[[" in str(source_note):
|
||||
flag("source-notes-wikilink", f"{path}: source_notes contains a wikilink '{source_note}'")
|
||||
|
||||
# Projects: folder<->status, stale active, duplicate slugs
|
||||
slug_homes: dict[str, list[str]] = {}
|
||||
for lifecycle in LIFECYCLES:
|
||||
files, _ = list_dir(f"projects/{lifecycle}")
|
||||
for filename in files:
|
||||
if filename in SKIP or not filename.endswith(".md"):
|
||||
continue
|
||||
slug = filename[:-3]
|
||||
slug_homes.setdefault(slug, []).append(lifecycle)
|
||||
text = get(f"projects/{lifecycle}/{filename}") or ""
|
||||
_, fm = parse_frontmatter(text)
|
||||
status = str(fm.get("status", "")).strip()
|
||||
if status and status != lifecycle:
|
||||
flag("folder/status", f"projects/{lifecycle}/{filename}: status='{status}' but folder='{lifecycle}'")
|
||||
if lifecycle == "active":
|
||||
updated = parse_date(fm.get("updated"))
|
||||
if updated and (TODAY - updated).days > STALE_DAYS:
|
||||
flag("stale-active", f"projects/active/{filename}: updated {updated} ({(TODAY - updated).days}d ago) — consider on-hold/")
|
||||
for slug, homes in slug_homes.items():
|
||||
if len(homes) > 1:
|
||||
flag("duplicate-slug", f"'{slug}' exists in {', '.join(homes)}")
|
||||
|
||||
# Daily notes: duplicate "## Agent Log" headings
|
||||
for path in all_files:
|
||||
if re.match(r"^journal/daily/.*\.md$", path):
|
||||
text = get(path) or ""
|
||||
count = len(re.findall(r"(?m)^## Agent Log\s*$", text))
|
||||
if count > 1:
|
||||
flag("duplicate-agent-log", f"{path}: {count} '## Agent Log' headings")
|
||||
|
||||
# Inbox: captures aging past INBOX_DAYS
|
||||
inbox = get("inbox/captures/inbox.md") or ""
|
||||
for line in inbox.splitlines():
|
||||
match = re.match(r"^\s*-\s*(\d{4}-\d{2}-\d{2})\b", line)
|
||||
date = parse_date(match.group(1)) if match else None
|
||||
if date and (TODAY - date).days > INBOX_DAYS:
|
||||
flag("aging-inbox", f"inbox capture {date} ({(TODAY - date).days}d): {line.strip()[:80]}")
|
||||
|
||||
# Scope freshness (drift detector)
|
||||
context = get("_agent/context/current-context.md")
|
||||
if context is not None:
|
||||
_, fm = parse_frontmatter(context)
|
||||
scope_updated = parse_date(fm.get("scope_updated"))
|
||||
if scope_updated is None:
|
||||
flag("scope-no-timestamp",
|
||||
"_agent/context/current-context.md: no scope_updated frontmatter — scope drift cannot be detected; add it (bootstrap.py) and switch scope via `echo.py scope set`")
|
||||
else:
|
||||
since = [
|
||||
path for path in all_files
|
||||
if (m := re.match(r"^_agent/sessions/(\d{4}-\d{2}-\d{2})", path))
|
||||
and (d := parse_date(m.group(1))) and d > scope_updated
|
||||
]
|
||||
if len(since) >= SCOPE_STALE_SESSIONS:
|
||||
flag("scope-stale",
|
||||
f"scope set {scope_updated}; {len(since)} session(s) logged since without a switch — confirm it still reflects current work (or run `echo.py scope set`)")
|
||||
|
||||
# ---- Graph health: broken wikilinks, orphans, index drift ----------------
|
||||
md_files = [p for p in all_files if p.endswith(".md")]
|
||||
md_set = set(md_files)
|
||||
basemap: dict[str, str] = {}
|
||||
for p in md_files:
|
||||
basemap.setdefault(idx_mod.slugify(p.rsplit("/", 1)[-1][:-3]), p)
|
||||
skip_link = re.compile(r"(^|/)(templates/|.*-template\.md$)")
|
||||
|
||||
def resolve_link(target: str):
|
||||
target = target.strip()
|
||||
if not target:
|
||||
return None
|
||||
if "/" in target:
|
||||
cand = target if target.endswith(".md") else target + ".md"
|
||||
return cand if cand in md_set else None
|
||||
return basemap.get(idx_mod.slugify(target))
|
||||
|
||||
text_cache: dict[str, str] = {}
|
||||
inbound: set[str] = set()
|
||||
for path in md_files:
|
||||
if path.rsplit("/", 1)[-1] in SKIP or skip_link.search(path):
|
||||
continue
|
||||
text = get(path) or ""
|
||||
text_cache[path] = text
|
||||
_, body = split_frontmatter(text)
|
||||
for t in links.all_wikilinks(body):
|
||||
tp = resolve_link(t)
|
||||
if tp is None:
|
||||
flag("broken-wikilink", f"{path}: [[{t}]] resolves to no note")
|
||||
else:
|
||||
inbound.add(tp)
|
||||
|
||||
def is_indexable(path: str) -> bool:
|
||||
# READMEs and templates are signposts, not entity notes (sweep skips them too).
|
||||
base = path.rsplit("/", 1)[-1]
|
||||
return (idx_mod.kind_for_path(path) is not None
|
||||
and base not in SKIP and not template_re.search(path))
|
||||
|
||||
# orphan: an entity note with no inbound links and an empty ## Related
|
||||
for path in md_files:
|
||||
if not is_indexable(path) or path in inbound:
|
||||
continue
|
||||
text = text_cache.get(path, get(path) or "")
|
||||
if not links.related_targets(text):
|
||||
flag("orphan", f"{path}: no inbound links and empty ## Related ({idx_mod.kind_for_path(path)}) — link it for recall")
|
||||
|
||||
# index drift: entries pointing at gone files, or indexable notes not yet indexed
|
||||
try:
|
||||
index = idx_mod.load()
|
||||
idx_paths = {e.get("path") for e in index.get("entities", {}).values()}
|
||||
for slug, e in index.get("entities", {}).items():
|
||||
if e.get("path") not in md_set:
|
||||
flag("index-stale", f"index entry '{slug}' -> {e.get('path')} no longer exists (run sweep.py)")
|
||||
for path in md_files:
|
||||
if is_indexable(path) and path not in idx_paths:
|
||||
flag("index-missing", f"{path}: indexable note absent from entities.json (run sweep.py)")
|
||||
except Exception as exc:
|
||||
flag("index-error", f"could not check entity index ({exc})")
|
||||
|
||||
if not violations:
|
||||
print("vault-lint: clean — all invariants hold.")
|
||||
return 0
|
||||
|
||||
print(f"vault-lint: {len(violations)} violation(s) found\n")
|
||||
labels = {
|
||||
"folder/status": "Folder <-> status mismatch",
|
||||
"duplicate-slug": "Duplicate slug across lifecycle folders",
|
||||
"frontmatter-wikilink": "Wikilink in frontmatter (breaks reading view)",
|
||||
"duplicate-agent-log": "Duplicate '## Agent Log' heading",
|
||||
"stale-active": f"Stale active project (updated > {STALE_DAYS}d)",
|
||||
"aging-inbox": f"Inbox capture aging (> {INBOX_DAYS}d)",
|
||||
"unknown-path": "Path matches no route in routing.json",
|
||||
"retired-path": "Write to a retired/dead path",
|
||||
"missing-frontmatter": "Missing required frontmatter field",
|
||||
"date-order": "updated earlier than created",
|
||||
"future-date": "updated date is in the future",
|
||||
"source-notes-wikilink": "Wikilink in source_notes (must be plain paths)",
|
||||
"routing-manifest": "routing.json problem",
|
||||
"scope-no-timestamp": "current-context has no scope_updated (drift undetectable)",
|
||||
"scope-stale": f"Scope may have drifted (>= {SCOPE_STALE_SESSIONS} sessions since last switch)",
|
||||
"broken-wikilink": "Wikilink resolves to no note (dead link)",
|
||||
"orphan": "Orphan entity note (no inbound links, empty Related)",
|
||||
"index-stale": "Entity-index entry points at a missing file",
|
||||
"index-missing": "Indexable note absent from the entity index",
|
||||
"index-error": "Entity-index check failed",
|
||||
}
|
||||
grouped: dict[str, list[str]] = {}
|
||||
for check, message in violations:
|
||||
grouped.setdefault(check, []).append(message)
|
||||
for check, messages in grouped.items():
|
||||
print(f"## {labels.get(check, check)}")
|
||||
for message in messages:
|
||||
print(f" - {message}")
|
||||
print()
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user