From db80f6e26c1bbc49aee1f92e59c3a2d39c7418d6 Mon Sep 17 00:00:00 2001 From: jp Date: Fri, 24 Apr 2026 09:07:46 -0700 Subject: [PATCH] fix: call quarantine_stale_hnsw() in make_client(); lower threshold to 5min MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit make_client() called _fix_blob_seq_ids but skipped quarantine_stale_hnsw, so every fresh process (stop hook, precompact hook, CLI) opened a drifted palace and segfaulted in chromadb_rust_bindings before any write-path protection could fire. #1062 wires the quarantine call at MCP server startup (covers long-lived server processes). This fix adds it to make_client() itself — the call site that all callers (server, hooks, CLI, tests) pass through — so every fresh PersistentClient open is protected regardless of entry point. Also lowers stale_seconds default from 3600 to 300: a 0.96h-drifted segment caused production segfaults before the 1h threshold fired. ChromaDB's HNSW flush cadence means legitimate drift is seconds to low minutes; 5min gives headroom without admitting clearly corrupt segments. --- mempalace/backends/chroma.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mempalace/backends/chroma.py b/mempalace/backends/chroma.py index c8d2f46..fee7d18 100644 --- a/mempalace/backends/chroma.py +++ b/mempalace/backends/chroma.py @@ -49,7 +49,7 @@ def _validate_where(where: Optional[dict]) -> None: stack.extend(x for x in v if isinstance(x, dict)) -def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 3600.0) -> list[str]: +def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 300.0) -> list[str]: """Rename HNSW segment dirs whose files are stale vs. chroma.sqlite3. When a ChromaDB 1.5.x PersistentClient opens a palace whose on-disk @@ -73,10 +73,12 @@ def quarantine_stale_hnsw(palace_path: str, stale_seconds: float = 3600.0) -> li original directory is renamed, not deleted, so recovery remains possible if the heuristic misfires. - The default threshold (1h) is deliberately conservative — ChromaDB's - HNSW flush cadence means legitimate drift is normally on the order of - seconds to minutes. A segment that is more than an hour out of date is - almost certainly in a "crashed mid-write" state. + The default threshold (5 min) is based on ChromaDB's HNSW flush + cadence — legitimate drift is normally on the order of seconds to + minutes. A segment more than 5 minutes out of date is almost certainly + in a "crashed mid-write" or "concurrent-write corrupted" state. The + previous 1h threshold was too conservative: 0.96h drift was observed + causing segfaults in production. Args: palace_path: path to the palace directory containing ``chroma.sqlite3`` @@ -544,6 +546,7 @@ class ChromaBackend(BaseBackend): :meth:`get_collection` which manages caching internally. """ _fix_blob_seq_ids(palace_path) + quarantine_stale_hnsw(palace_path) return chromadb.PersistentClient(path=palace_path) @staticmethod