79 lines
3.3 KiB
Python
79 lines
3.3 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""echo_stem.py — a deliberately light, dependency-free English stemmer. [2.3]
|
||
|
|
|
||
|
|
BM25 is purely lexical: "deploy", "deployed", and "deployment" are three unrelated
|
||
|
|
terms to it, so a paraphrased recall query misses notes stored with a different
|
||
|
|
inflection. This Porter-LITE stemmer conflates the common inflection families while
|
||
|
|
staying conservative — when in doubt it leaves the token alone, because an
|
||
|
|
over-stemmed index silently merges unrelated words (far worse than a missed match).
|
||
|
|
|
||
|
|
Applied by echo_recall.tokenize() at BOTH index and query time (recall-index
|
||
|
|
schema 3; older indexes are discarded and rebuilt). Pure function, unit-tested in
|
||
|
|
test_echo_client.py.
|
||
|
|
|
||
|
|
Design notes:
|
||
|
|
* plural / -ed / -ing families first (with the classic restore rules: doubled
|
||
|
|
consonant undoubling, at/bl/iz +e, cvc +e), then one derivational suffix pass,
|
||
|
|
then y->i and final-e normalization so "navigate"/"navigation"/"navigating"
|
||
|
|
all land on "navigat".
|
||
|
|
* every rule guards a minimum remaining stem length — "sing" is not "s".
|
||
|
|
"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import re
|
||
|
|
|
||
|
|
_VOWEL = re.compile(r"[aeiouy]")
|
||
|
|
_DOUBLE = re.compile(r"([^aeiouylsz])\1$")
|
||
|
|
_CVC = re.compile(r"[^aeiou][aeiouy][^aeiouwxy]$")
|
||
|
|
|
||
|
|
# One derivational pass, most-specific first. Replacements keep families together:
|
||
|
|
# navigational/navigation -> navigat · saturation/saturated -> saturat ·
|
||
|
|
# deployment/deployed -> deploy · usefulness/useful -> use... (guarded by length).
|
||
|
|
_SUFFIXES = (
|
||
|
|
("ization", "ize"), ("ational", "ate"), ("fulness", "ful"), ("ousness", "ous"),
|
||
|
|
("iveness", "ive"), ("tional", "t"), ("biliti", "ble"), ("ements", ""),
|
||
|
|
("ment", ""), ("ness", ""), ("tion", "t"), ("sion", "s"), ("ance", ""),
|
||
|
|
("ence", ""), ("able", ""), ("ible", ""), ("ally", "al"), ("ously", "ous"),
|
||
|
|
("ively", "ive"), ("ly", ""),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def stem(t: str) -> str:
|
||
|
|
"""Stem one lowercase token. Never raises; returns the token unchanged when no
|
||
|
|
rule applies safely."""
|
||
|
|
if len(t) <= 3 or not t.isascii():
|
||
|
|
return t
|
||
|
|
# --- plurals -------------------------------------------------------------
|
||
|
|
if t.endswith("sses"):
|
||
|
|
t = t[:-2]
|
||
|
|
elif t.endswith("ies") and len(t) > 4:
|
||
|
|
t = t[:-3] + "i"
|
||
|
|
elif t.endswith("s") and not t.endswith(("ss", "us", "is")):
|
||
|
|
t = t[:-1]
|
||
|
|
# --- -ed / -ing (with restore rules) --------------------------------------
|
||
|
|
for suf in ("ingly", "edly", "ing", "ed"):
|
||
|
|
if t.endswith(suf):
|
||
|
|
base = t[: -len(suf)]
|
||
|
|
if len(base) >= 3 and _VOWEL.search(base):
|
||
|
|
t = base
|
||
|
|
if t.endswith(("at", "bl", "iz")):
|
||
|
|
t += "e"
|
||
|
|
elif _DOUBLE.search(t):
|
||
|
|
t = t[:-1]
|
||
|
|
elif len(t) == 3 and _CVC.search(t):
|
||
|
|
t += "e"
|
||
|
|
break
|
||
|
|
# --- one derivational suffix pass -----------------------------------------
|
||
|
|
for suf, rep in _SUFFIXES:
|
||
|
|
if t.endswith(suf):
|
||
|
|
base = t[: -len(suf)]
|
||
|
|
if len(base) >= 3 and len(base + rep) >= 3:
|
||
|
|
t = base + rep
|
||
|
|
break
|
||
|
|
# --- normalize: y->i (so penalty/penalties agree), then drop a final e -----
|
||
|
|
if len(t) > 3 and t.endswith("y") and t[-2] not in "aeiou":
|
||
|
|
t = t[:-1] + "i"
|
||
|
|
if len(t) > 4 and t.endswith("e"):
|
||
|
|
t = t[:-1]
|
||
|
|
return t
|