mempalace/mempalace/sources/base.py

"""Source adapter contract for MemPalace (RFC 002).

Mirrors what ``mempalace/backends/base.py`` does for the write side: it defines
the read-side surface every source adapter must implement. A source adapter
extracts content from a specific origin (filesystem, git, Slack, Cursor …) and
yields typed records (``SourceItemMetadata`` / ``DrawerRecord``) that core
routes into the palace.

This module is spec scaffolding. The first-party miners (``mempalace/miner.py``
and ``mempalace/convo_miner.py``) are migrated onto it in a follow-up PR;
in this PR we publish the contract so third-party adapters can begin building
against a stable surface.

See ``docs/rfcs/002-source-adapter-plugin-spec.md`` for the authoritative
spec text.
"""

from __future__ import annotations

from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, ClassVar, Iterator, Literal, Optional

if TYPE_CHECKING:
    from .context import PalaceContext  # noqa: F401  (used in string annotation)


# ---------------------------------------------------------------------------
# Errors
# ---------------------------------------------------------------------------


class SourceAdapterError(Exception):
    """Base class for every source-adapter error raised by core."""


class SourceNotFoundError(SourceAdapterError):
    """Raised when a ``SourceRef`` does not resolve to a readable source."""


class AuthRequiredError(SourceAdapterError):
    """Raised when an adapter needs credentials that were not provided.

    The message MUST name the env vars (or other supported mechanism) the
    operator needs to set.
    """


class AdapterClosedError(SourceAdapterError):
    """Raised when an adapter method is called after ``close()``."""


class TransformationViolationError(SourceAdapterError):
    """Raised by the conformance suite when round-tripping a drawer requires
    an undeclared transformation (RFC 002 §7.2–7.3)."""


class SchemaConformanceError(SourceAdapterError):
    """Raised when a ``DrawerRecord.metadata`` violates the adapter schema
    returned by :meth:`BaseSourceAdapter.describe_schema`."""


# ---------------------------------------------------------------------------
# Value objects
# ---------------------------------------------------------------------------


@dataclass(frozen=True)
class SourceRef:
    """A handle to the source a user wants to ingest.

    ``local_path`` is for filesystem-rooted sources (project dir, mbox file).
    ``uri`` is for URL-like references (``github.com/org/repo``,
    ``slack://workspace/channel``).
    ``options`` carries adapter-specific non-secret config. Secrets MUST NOT
    be placed here; see §4.2.
    """

    local_path: Optional[str] = None
    uri: Optional[str] = None
    options: dict = field(default_factory=dict)


@dataclass(frozen=True)
class RouteHint:
    """Adapter-supplied routing hint (RFC 002 §2.5)."""

    wing: Optional[str] = None
    room: Optional[str] = None
    hall: Optional[str] = None


@dataclass(frozen=True)
class SourceItemMetadata:
    """Lightweight pointer yielded before drawers for lazy-fetch adapters.

    Core inspects ``version`` via :meth:`BaseSourceAdapter.is_current` to
    decide whether to skip extraction; an adapter that responds positively
    stops yielding drawers for this item and moves to the next.
    """

    source_file: str
    version: str
    size_hint: Optional[int] = None
    route_hint: Optional[RouteHint] = None


@dataclass(frozen=True)
class DrawerRecord:
    """One drawer's worth of extracted content plus flat metadata.

    ``metadata`` values MUST be flat scalars (``str``/``int``/``float``/``bool``)
    per RFC 001 §1.4 — the chroma constraint. Nested data belongs on the
    knowledge graph (§5.5) or in a declared ``json_string`` field (§5.4).
    """

    content: str
    source_file: str
    chunk_index: int = 0
    metadata: dict = field(default_factory=dict)
    route_hint: Optional[RouteHint] = None


@dataclass(frozen=True)
class SourceSummary:
    """High-level description of a source returned by :meth:`source_summary`."""

    description: str
    item_count: Optional[int] = None


IngestMode = Literal["chunked_content", "whole_record", "metadata_only"]


@dataclass(frozen=True)
class FieldSpec:
    """Declared shape of a single per-adapter metadata field (§5.2)."""

    type: Literal["string", "int", "float", "bool", "delimiter_joined_string", "json_string"]
    required: bool
    description: str
    indexed: bool = False
    delimiter: str = ";"
    json_schema: Optional[dict] = None


@dataclass(frozen=True)
class AdapterSchema:
    """The per-adapter metadata schema returned by :meth:`describe_schema`."""

    fields: dict[str, FieldSpec]
    version: str


# The union type adapters yield from ``ingest``.
IngestResult = object  # intentionally broad; runtime checks in core


# ---------------------------------------------------------------------------
# Adapter contract
# ---------------------------------------------------------------------------


class BaseSourceAdapter(ABC):
    """Long-lived adapter serving many ``SourceRef`` invocations (RFC 002 §2).

    Instances are lightweight on construction — no I/O, no network, no
    credential fetch. All work is deferred to :meth:`ingest`. Instances are
    thread-safe for concurrent ``ingest`` calls across different ``SourceRef``
    values (v1 serializes within a single ``SourceRef``).

    Class attributes form the adapter's identity contract:

    * ``name`` — stable adapter name used for registration and drawer metadata.
    * ``adapter_version`` — adapter's own version, independent of
      ``spec_version``. Recorded on every drawer so re-extract workflows can
      target drawers from a known-buggy adapter version.
    * ``capabilities`` — free-form tokens; core inspects a documented subset.
    * ``supported_modes`` — subset of ``chunked_content``, ``whole_record``,
      ``metadata_only``.
    * ``declared_transformations`` — set of transformation names the adapter
      applies to source bytes. The empty set marks a byte-preserving adapter.
    * ``default_privacy_class`` — privacy class level (§6) applied unless the
      palace config overrides it.
    """

    name: ClassVar[str]
    spec_version: ClassVar[str] = "1.0"
    adapter_version: ClassVar[str] = "0.0.0"
    capabilities: ClassVar[frozenset[str]] = frozenset()
    supported_modes: ClassVar[frozenset[str]] = frozenset({"chunked_content"})
    declared_transformations: ClassVar[frozenset[str]] = frozenset()
    default_privacy_class: ClassVar[str] = "pii_potential"

    # ------------------------------------------------------------------
    # Required methods
    # ------------------------------------------------------------------

    @abstractmethod
    def ingest(
        self,
        *,
        source: SourceRef,
        palace: "PalaceContext",
    ) -> Iterator[IngestResult]:
        """Enumerate and extract content from a source.

        Yields a stream of ``SourceItemMetadata`` and ``DrawerRecord`` values.
        Lazy adapters yield ``SourceItemMetadata`` ahead of the drawers for
        that item so core can check :meth:`is_current` before committing to
        the fetch. Eager adapters MAY interleave freely.
        """

    @abstractmethod
    def describe_schema(self) -> AdapterSchema:
        """Declare the structured metadata this adapter attaches.

        The returned schema MUST be stable for a given ``adapter_version``.
        Enterprises index on it; core uses it to validate adapter output.
        """

    # ------------------------------------------------------------------
    # Optional methods with default implementations
    # ------------------------------------------------------------------

    def is_current(
        self,
        *,
        item: SourceItemMetadata,
        existing_metadata: Optional[dict],
    ) -> bool:
        """Return True if the palace already has an up-to-date copy of ``item``.

        Default: always returns False (re-extract every time). Adapters
        advertising ``supports_incremental`` MUST override.
        """
        return False

    def source_summary(self, *, source: SourceRef) -> SourceSummary:
        """Describe a source without extracting."""
        return SourceSummary(description=self.name)

    def close(self) -> None:
        """Release any resources the adapter holds. Default: no-op."""
        return None