Merge pull request #66 from MARUCIE/fix/sqlite-batch-reads

fix: batch ChromaDB reads to avoid SQLite variable limit

mempalace/cli.py

@@ -253,20 +253,31 @@ def cmd_compress(args):
         print("  Run: mempalace init <dir> then mempalace mine <dir>")
         sys.exit(1)
 
-    # Query drawers in the wing
+    # Query drawers in batches to avoid SQLite variable limit (~999)
     where = {"wing": args.wing} if args.wing else None
+    _BATCH = 500
+    docs, metas, ids = [], [], []
+    offset = 0
+    while True:
         try:
-        kwargs = {"include": ["documents", "metadatas"]}
+            kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
             if where:
                 kwargs["where"] = where
-        results = col.get(**kwargs)
+            batch = col.get(**kwargs)
         except Exception as e:
+            if not docs:
                 print(f"\n  Error reading drawers: {e}")
                 sys.exit(1)
-
+            break
-    docs = results["documents"]
+        batch_docs = batch.get("documents", [])
-    metas = results["metadatas"]
+        if not batch_docs:
-    ids = results["ids"]
+            break
+        docs.extend(batch_docs)
+        metas.extend(batch.get("metadatas", []))
+        ids.extend(batch.get("ids", []))
+        offset += len(batch_docs)
+        if len(batch_docs) < _BATCH:
+            break
 
     if not docs:
         wing_label = f" in wing '{args.wing}'" if args.wing else ""

mempalace/layers.py

@@ -96,18 +96,27 @@ class Layer1:
         except Exception:
             return "## L1 — No palace found. Run: mempalace mine <dir>"
 
-        # Fetch all drawers (with optional wing filter)
+        # Fetch all drawers in batches to avoid SQLite variable limit (~999)
-        kwargs = {"include": ["documents", "metadatas"]}
+        _BATCH = 500
+        docs, metas = [], []
+        offset = 0
+        while True:
+            kwargs = {"include": ["documents", "metadatas"], "limit": _BATCH, "offset": offset}
             if self.wing:
                 kwargs["where"] = {"wing": self.wing}
-
             try:
-            results = col.get(**kwargs)
+                batch = col.get(**kwargs)
             except Exception:
-            return "## L1 — No drawers found."
+                break
-
+            batch_docs = batch.get("documents", [])
-        docs = results.get("documents", [])
+            batch_metas = batch.get("metadatas", [])
-        metas = results.get("metadatas", [])
+            if not batch_docs:
+                break
+            docs.extend(batch_docs)
+            metas.extend(batch_metas)
+            offset += len(batch_docs)
+            if len(batch_docs) < _BATCH:
+                break
 
         if not docs:
             return "## L1 — No memories yet."