MSL
30a431924b
Root cause: when multiple agents mine simultaneously, both pass file_already_mined() check, both delete+insert the same file's drawers, creating duplicates or losing data. Fix: mine_lock() in palace.py — cross-platform file lock (fcntl on Unix, msvcrt on Windows). Both miner.py and convo_miner.py now lock per-file during the delete+insert cycle and re-check after acquiring the lock. Tested: - Lock acquires and releases correctly - Second agent blocks until first releases (0.25s wait) - 33/33 existing tests pass - Cross-platform: fcntl (macOS/Linux), msvcrt (Windows) Based on v3.2.0 tag. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
111 lines
2.9 KiB
Python
111 lines
2.9 KiB
Python
"""
|
|
palace.py — Shared palace operations.
|
|
|
|
Consolidates collection access patterns used by both miners and the MCP server.
|
|
"""
|
|
|
|
import contextlib
|
|
import hashlib
|
|
import os
|
|
|
|
from .backends.chroma import ChromaBackend
|
|
|
|
SKIP_DIRS = {
|
|
".git",
|
|
"node_modules",
|
|
"__pycache__",
|
|
".venv",
|
|
"venv",
|
|
"env",
|
|
"dist",
|
|
"build",
|
|
".next",
|
|
"coverage",
|
|
".mempalace",
|
|
".ruff_cache",
|
|
".mypy_cache",
|
|
".pytest_cache",
|
|
".cache",
|
|
".tox",
|
|
".nox",
|
|
".idea",
|
|
".vscode",
|
|
".ipynb_checkpoints",
|
|
".eggs",
|
|
"htmlcov",
|
|
"target",
|
|
}
|
|
|
|
_DEFAULT_BACKEND = ChromaBackend()
|
|
|
|
|
|
def get_collection(
|
|
palace_path: str,
|
|
collection_name: str = "mempalace_drawers",
|
|
create: bool = True,
|
|
):
|
|
"""Get the palace collection through the backend layer."""
|
|
return _DEFAULT_BACKEND.get_collection(
|
|
palace_path,
|
|
collection_name=collection_name,
|
|
create=create,
|
|
)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def mine_lock(source_file: str):
|
|
"""Cross-platform file lock for mine operations.
|
|
|
|
Prevents multiple agents from mining the same file simultaneously,
|
|
which causes duplicate drawers when the delete+insert cycle interleaves.
|
|
"""
|
|
lock_dir = os.path.join(os.path.expanduser("~"), ".mempalace", "locks")
|
|
os.makedirs(lock_dir, exist_ok=True)
|
|
lock_path = os.path.join(
|
|
lock_dir, hashlib.sha256(source_file.encode()).hexdigest()[:16] + ".lock"
|
|
)
|
|
|
|
lf = open(lock_path, "w")
|
|
try:
|
|
if os.name == "nt":
|
|
import msvcrt
|
|
msvcrt.locking(lf.fileno(), msvcrt.LK_LOCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(lf, fcntl.LOCK_EX)
|
|
yield
|
|
finally:
|
|
try:
|
|
if os.name == "nt":
|
|
import msvcrt
|
|
msvcrt.locking(lf.fileno(), msvcrt.LK_UNLCK, 1)
|
|
else:
|
|
import fcntl
|
|
fcntl.flock(lf, fcntl.LOCK_UN)
|
|
except Exception:
|
|
pass
|
|
lf.close()
|
|
|
|
|
|
def file_already_mined(collection, source_file: str, check_mtime: bool = False) -> bool:
|
|
"""Check if a file has already been filed in the palace.
|
|
|
|
When check_mtime=True (used by project miner), returns False if the file
|
|
has been modified since it was last mined, so it gets re-mined.
|
|
When check_mtime=False (used by convo miner), just checks existence.
|
|
"""
|
|
try:
|
|
results = collection.get(where={"source_file": source_file}, limit=1)
|
|
if not results.get("ids"):
|
|
return False
|
|
if check_mtime:
|
|
stored_meta = results.get("metadatas", [{}])[0]
|
|
stored_mtime = stored_meta.get("source_mtime")
|
|
if stored_mtime is None:
|
|
return False
|
|
current_mtime = os.path.getmtime(source_file)
|
|
return abs(float(stored_mtime) - current_mtime) < 0.001
|
|
return True
|
|
except Exception:
|
|
return False
|