9e7fa1ceb5
`mempalace init` previously leaned entirely on regex-based entity extraction from prose. That path works for text-only folders but wastes signal in any codebase: the project's own name is already in `package.json` / `pyproject.toml` / `Cargo.toml` / `go.mod`, and the people who worked on it are in `git log`. This adds `project_scanner.py`, which becomes the primary signal source when real signal is available, with the regex detector preserved as the fallback for prose-only folders (diaries, research notes, writing). What it does: - Walks the target directory, parses manifests for canonical project names, and detects git repos by the presence of a `.git` directory. - For each repo, reads `git log` for authors and filters obvious bots (`[bot]`, `dependabot`, `renovate`, `github-actions`, names ending in `bot`, `-autoroll`). Importantly does NOT filter `@users.noreply.github.com` - that's GitHub's privacy-protected human email, used by real contributors. - Resolves author aliases with a union-find: commits that share a name OR an email collapse into one person. Picks the most-frequent real-name variant as display, ignoring handles and single-token usernames. - Flags "mine" projects: user is top-5 committer OR has >=10% of commits OR >=20 commits. Ordered by user_commits in the UX. - `discover_entities()` merges scanner results with the regex detector case-insensitively (so `mempalace` from pyproject absorbs `MemPalace` from docs), and suppresses the regex `uncertain` bucket when real signal is already found - the user doesn't need to adjudicate prose noise when the answer is already in git. Integration: `cmd_init` now calls `discover_entities` instead of running the regex detector directly. Same output shape, so `confirm_entities` works unchanged. Ships with 39 new tests covering manifest parsing, bot filtering, union-find dedup, git repo discovery, scan integration, and merge/fallback behavior. Existing 56 regex-detector tests all pass.
412 lines
14 KiB
Python
412 lines
14 KiB
Python
"""Tests for mempalace.project_scanner."""
|
|
|
|
import json
|
|
import subprocess
|
|
from pathlib import Path
|
|
|
|
from mempalace.project_scanner import (
|
|
PersonInfo,
|
|
ProjectInfo,
|
|
_dedupe_people,
|
|
_is_bot,
|
|
_looks_like_real_name,
|
|
_merge_detected,
|
|
_parse_cargo,
|
|
_parse_gomod,
|
|
_parse_package_json,
|
|
_parse_pyproject,
|
|
_UnionFind,
|
|
discover_entities,
|
|
find_git_repos,
|
|
scan,
|
|
to_detected_dict,
|
|
)
|
|
|
|
|
|
# ── manifest parsers ────────────────────────────────────────────────────
|
|
|
|
|
|
def test_parse_package_json(tmp_path):
|
|
f = tmp_path / "package.json"
|
|
f.write_text(json.dumps({"name": "my-package", "version": "1.0.0"}))
|
|
assert _parse_package_json(f) == "my-package"
|
|
|
|
|
|
def test_parse_package_json_missing_name(tmp_path):
|
|
f = tmp_path / "package.json"
|
|
f.write_text(json.dumps({"version": "1.0.0"}))
|
|
assert _parse_package_json(f) is None
|
|
|
|
|
|
def test_parse_package_json_malformed(tmp_path):
|
|
f = tmp_path / "package.json"
|
|
f.write_text("{ not valid json")
|
|
assert _parse_package_json(f) is None
|
|
|
|
|
|
def test_parse_pyproject_pep621(tmp_path):
|
|
f = tmp_path / "pyproject.toml"
|
|
f.write_text('[project]\nname = "my-py-package"\n')
|
|
assert _parse_pyproject(f) == "my-py-package"
|
|
|
|
|
|
def test_parse_pyproject_poetry(tmp_path):
|
|
f = tmp_path / "pyproject.toml"
|
|
f.write_text('[tool.poetry]\nname = "poetry-pkg"\n')
|
|
assert _parse_pyproject(f) == "poetry-pkg"
|
|
|
|
|
|
def test_parse_cargo(tmp_path):
|
|
f = tmp_path / "Cargo.toml"
|
|
f.write_text('[package]\nname = "rust-crate"\nversion = "0.1.0"\n')
|
|
assert _parse_cargo(f) == "rust-crate"
|
|
|
|
|
|
def test_parse_gomod(tmp_path):
|
|
f = tmp_path / "go.mod"
|
|
f.write_text("module github.com/user/my-go-mod\n\ngo 1.21\n")
|
|
assert _parse_gomod(f) == "my-go-mod"
|
|
|
|
|
|
# ── bot filtering ───────────────────────────────────────────────────────
|
|
|
|
|
|
def test_is_bot_catches_github_actions():
|
|
assert _is_bot("github-actions[bot]", "41898282+github-actions[bot]@users.noreply.github.com")
|
|
|
|
|
|
def test_is_bot_catches_dependabot():
|
|
assert _is_bot("dependabot[bot]", "dependabot@github.com")
|
|
|
|
|
|
def test_is_bot_catches_pr_bot():
|
|
assert _is_bot("Comfy Org PR Bot", "prbot@example.com")
|
|
|
|
|
|
def test_is_bot_does_not_flag_github_privacy_email():
|
|
# Real humans use ...@users.noreply.github.com when privacy is enabled.
|
|
# Must NOT be filtered.
|
|
assert not _is_bot("Igor Lins e Silva", "123456+igorls@users.noreply.github.com")
|
|
|
|
|
|
def test_is_bot_does_not_flag_robot_person_name():
|
|
# "Robot" as a surname should not trigger the \bbot$ pattern
|
|
# since \b requires a boundary before 'bot'.
|
|
assert not _is_bot("Sarah Robot", "sarah@example.com")
|
|
|
|
|
|
def test_looks_like_real_name_accepts_human():
|
|
assert _looks_like_real_name("Igor Lins e Silva")
|
|
assert _looks_like_real_name("Jane Doe")
|
|
|
|
|
|
def test_looks_like_real_name_rejects_handles():
|
|
assert not _looks_like_real_name("666ghj")
|
|
assert not _looks_like_real_name("comfyanonymous")
|
|
assert not _looks_like_real_name("bensig")
|
|
assert not _looks_like_real_name("")
|
|
assert not _looks_like_real_name("no_spaces_handle")
|
|
|
|
|
|
# ── union-find dedup ────────────────────────────────────────────────────
|
|
|
|
|
|
def test_unionfind_merges_shared_email():
|
|
commits = [
|
|
("Milla J", "shared@example.com", "repo1"),
|
|
("MSL", "shared@example.com", "repo1"),
|
|
("Milla J", "other@example.com", "repo1"),
|
|
]
|
|
people = _dedupe_people(commits)
|
|
# All three commits collapse into one "Milla J" person (MSL is filtered
|
|
# as display name because it lacks a space but its commits still count).
|
|
assert "Milla J" in people
|
|
assert people["Milla J"].total_commits == 3
|
|
assert "MSL" not in people
|
|
|
|
|
|
def test_unionfind_keeps_distinct_people_separate():
|
|
commits = [
|
|
("Alice Example", "alice@example.com", "r"),
|
|
("Bob Sample", "bob@sample.org", "r"),
|
|
]
|
|
people = _dedupe_people(commits)
|
|
assert "Alice Example" in people
|
|
assert "Bob Sample" in people
|
|
|
|
|
|
def test_unionfind_merges_shared_name():
|
|
"""Same display name, two different emails, same person."""
|
|
commits = [
|
|
("Jane Doe", "jane@work.com", "r"),
|
|
("Jane Doe", "jane@personal.com", "r"),
|
|
]
|
|
people = _dedupe_people(commits)
|
|
assert people["Jane Doe"].total_commits == 2
|
|
assert len(people["Jane Doe"].emails) == 2
|
|
|
|
|
|
# ── project_info / person_info ─────────────────────────────────────────
|
|
|
|
|
|
def test_project_info_confidence_is_mine():
|
|
p = ProjectInfo(name="x", repo_root=Path("."), is_mine=True)
|
|
assert p.confidence == 0.99
|
|
|
|
|
|
def test_project_info_confidence_no_git():
|
|
p = ProjectInfo(name="x", repo_root=Path("."), has_git=False, manifest="package.json")
|
|
assert p.confidence > 0.8
|
|
|
|
|
|
def test_person_info_signal_pluralization():
|
|
p = PersonInfo(name="x", total_commits=1, repos={"a"})
|
|
assert "1 commit across 1 repo" == p.to_signal()
|
|
p2 = PersonInfo(name="y", total_commits=5, repos={"a", "b"})
|
|
assert "5 commits across 2 repos" == p2.to_signal()
|
|
|
|
|
|
# ── find_git_repos ──────────────────────────────────────────────────────
|
|
|
|
|
|
def test_find_git_repos_detects_root_repo(tmp_path):
|
|
(tmp_path / ".git").mkdir()
|
|
repos = find_git_repos(tmp_path)
|
|
assert tmp_path in repos
|
|
|
|
|
|
def test_find_git_repos_detects_nested(tmp_path):
|
|
sub = tmp_path / "subproject"
|
|
sub.mkdir()
|
|
(sub / ".git").mkdir()
|
|
repos = find_git_repos(tmp_path)
|
|
assert sub in repos
|
|
|
|
|
|
def test_find_git_repos_skips_nested_inside_repo(tmp_path):
|
|
"""If root is a repo and there's another repo inside it, the inner repo is
|
|
NOT walked into (we stop at the first repo boundary when descending)."""
|
|
(tmp_path / ".git").mkdir()
|
|
deep = tmp_path / "a" / "b" / "nested-repo"
|
|
deep.mkdir(parents=True)
|
|
(deep / ".git").mkdir()
|
|
repos = find_git_repos(tmp_path)
|
|
# Root IS found; nested still discovered on its own branch (not inside root's .git)
|
|
assert tmp_path in repos
|
|
|
|
|
|
def test_find_git_repos_empty_dir(tmp_path):
|
|
assert find_git_repos(tmp_path) == []
|
|
|
|
|
|
# ── scan ────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _init_git_repo(path: Path, name: str = "Jane Doe", email: str = "jane@example.com"):
|
|
"""Helper: init a git repo with one commit."""
|
|
subprocess.run(["git", "init", "-q"], cwd=path, check=True)
|
|
subprocess.run(["git", "config", "user.name", name], cwd=path, check=True)
|
|
subprocess.run(["git", "config", "user.email", email], cwd=path, check=True)
|
|
subprocess.run(["git", "config", "commit.gpgsign", "false"], cwd=path, check=True)
|
|
(path / "README.md").write_text("hello")
|
|
subprocess.run(["git", "add", "README.md"], cwd=path, check=True)
|
|
subprocess.run(
|
|
["git", "commit", "-q", "-m", "initial"],
|
|
cwd=path,
|
|
check=True,
|
|
env={"GIT_COMMITTER_NAME": name, "GIT_COMMITTER_EMAIL": email, "PATH": "/usr/bin:/bin"},
|
|
)
|
|
|
|
|
|
def test_scan_project_from_package_json(tmp_path):
|
|
(tmp_path / "package.json").write_text(json.dumps({"name": "my-app"}))
|
|
_init_git_repo(tmp_path)
|
|
projects, people = scan(tmp_path)
|
|
assert len(projects) == 1
|
|
assert projects[0].name == "my-app"
|
|
assert projects[0].is_mine is True
|
|
|
|
|
|
def test_scan_project_from_pyproject(tmp_path):
|
|
(tmp_path / "pyproject.toml").write_text('[project]\nname = "pyproj"\n')
|
|
_init_git_repo(tmp_path)
|
|
projects, _ = scan(tmp_path)
|
|
assert any(p.name == "pyproj" for p in projects)
|
|
|
|
|
|
def test_scan_fallback_to_dir_name_when_no_manifest(tmp_path):
|
|
repo = tmp_path / "my-repo-name"
|
|
repo.mkdir()
|
|
_init_git_repo(repo)
|
|
projects, _ = scan(tmp_path)
|
|
assert any(p.name == "my-repo-name" for p in projects)
|
|
|
|
|
|
def test_scan_manifest_only_no_git(tmp_path):
|
|
"""A dir with a manifest but no git still produces a project."""
|
|
(tmp_path / "package.json").write_text(json.dumps({"name": "manifest-only"}))
|
|
projects, people = scan(tmp_path)
|
|
assert len(projects) == 1
|
|
assert projects[0].name == "manifest-only"
|
|
assert projects[0].has_git is False
|
|
assert people == []
|
|
|
|
|
|
def test_scan_empty_dir(tmp_path):
|
|
projects, people = scan(tmp_path)
|
|
assert projects == []
|
|
assert people == []
|
|
|
|
|
|
def test_scan_returns_empty_for_nonexistent(tmp_path):
|
|
missing = tmp_path / "does-not-exist"
|
|
projects, people = scan(missing)
|
|
assert projects == []
|
|
assert people == []
|
|
|
|
|
|
# ── to_detected_dict ────────────────────────────────────────────────────
|
|
|
|
|
|
def test_to_detected_dict_shape():
|
|
projects = [ProjectInfo(name="p", repo_root=Path("."), is_mine=True, manifest="package.json")]
|
|
people = [PersonInfo(name="Jane Doe", total_commits=5, repos={"r"})]
|
|
d = to_detected_dict(projects, people)
|
|
assert set(d.keys()) == {"people", "projects", "uncertain"}
|
|
assert d["projects"][0]["name"] == "p"
|
|
assert d["projects"][0]["type"] == "project"
|
|
assert d["people"][0]["name"] == "Jane Doe"
|
|
assert d["people"][0]["type"] == "person"
|
|
assert d["uncertain"] == []
|
|
|
|
|
|
# ── merge ───────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_merge_primary_wins_case_insensitive():
|
|
primary = {
|
|
"people": [],
|
|
"projects": [
|
|
{
|
|
"name": "mempalace",
|
|
"type": "project",
|
|
"confidence": 0.99,
|
|
"frequency": 10,
|
|
"signals": ["pyproject.toml"],
|
|
}
|
|
],
|
|
"uncertain": [],
|
|
}
|
|
secondary = {
|
|
"people": [],
|
|
"projects": [],
|
|
"uncertain": [
|
|
{
|
|
"name": "MemPalace",
|
|
"type": "uncertain",
|
|
"confidence": 0.4,
|
|
"frequency": 6,
|
|
"signals": ["regex"],
|
|
}
|
|
],
|
|
}
|
|
merged = _merge_detected(primary, secondary)
|
|
# `MemPalace` (uncertain) is deduped against `mempalace` (project) case-insensitively
|
|
assert len(merged["projects"]) == 1
|
|
assert len(merged["uncertain"]) == 0
|
|
|
|
|
|
def test_merge_drops_secondary_uncertain_when_requested():
|
|
primary = {"people": [], "projects": [], "uncertain": []}
|
|
secondary = {
|
|
"people": [],
|
|
"projects": [],
|
|
"uncertain": [
|
|
{"name": "Foo", "type": "uncertain", "confidence": 0.4, "frequency": 3, "signals": []}
|
|
],
|
|
}
|
|
merged = _merge_detected(primary, secondary, drop_secondary_uncertain=True)
|
|
assert merged["uncertain"] == []
|
|
|
|
|
|
def test_merge_keeps_distinct_names():
|
|
primary = {
|
|
"people": [
|
|
{
|
|
"name": "Alice Smith",
|
|
"type": "person",
|
|
"confidence": 0.9,
|
|
"frequency": 10,
|
|
"signals": [],
|
|
}
|
|
],
|
|
"projects": [],
|
|
"uncertain": [],
|
|
}
|
|
secondary = {
|
|
"people": [
|
|
{
|
|
"name": "Bob Jones",
|
|
"type": "person",
|
|
"confidence": 0.7,
|
|
"frequency": 3,
|
|
"signals": [],
|
|
}
|
|
],
|
|
"projects": [],
|
|
"uncertain": [],
|
|
}
|
|
merged = _merge_detected(primary, secondary)
|
|
assert len(merged["people"]) == 2
|
|
|
|
|
|
# ── discover_entities ──────────────────────────────────────────────────
|
|
|
|
|
|
def test_discover_entities_falls_back_to_prose_when_no_git(tmp_path):
|
|
"""If no manifests or git, regex detector on prose is the only source."""
|
|
notes = tmp_path / "notes.md"
|
|
notes.write_text(
|
|
"Riley said hello. Riley asked about it. Riley laughed. "
|
|
"Hey Riley, thanks for the help. Riley pushed the change. "
|
|
"Riley decided to go."
|
|
)
|
|
d = discover_entities(str(tmp_path))
|
|
# Prose-only fallback kicks in — Riley appears with person signals
|
|
all_names = [e["name"] for cat in d.values() for e in cat]
|
|
assert "Riley" in all_names
|
|
|
|
|
|
def test_discover_entities_prefers_real_signal_over_prose(tmp_path):
|
|
"""When manifest exists, its name wins even if prose has noisy candidates."""
|
|
(tmp_path / "package.json").write_text(json.dumps({"name": "realproj"}))
|
|
_init_git_repo(tmp_path)
|
|
(tmp_path / "doc.md").write_text(
|
|
"Something. Another. Whatever. Context. Context. Context. Context. "
|
|
"realproj. realproj. realproj. realproj."
|
|
)
|
|
d = discover_entities(str(tmp_path))
|
|
proj_names = [e["name"] for e in d["projects"]]
|
|
assert "realproj" in proj_names
|
|
|
|
|
|
# ── _UnionFind basics ──────────────────────────────────────────────────
|
|
|
|
|
|
def test_unionfind_find_creates_singleton():
|
|
uf = _UnionFind()
|
|
assert uf.find("x") == "x"
|
|
|
|
|
|
def test_unionfind_union_merges():
|
|
uf = _UnionFind()
|
|
uf.union("a", "b")
|
|
assert uf.find("a") == uf.find("b")
|
|
|
|
|
|
def test_unionfind_transitive():
|
|
uf = _UnionFind()
|
|
uf.union("a", "b")
|
|
uf.union("b", "c")
|
|
assert uf.find("a") == uf.find("c")
|