Jason Stedwell
360 lines
10 KiB
JSON
360 lines
10 KiB
JSON
{
|
|
"env": {
|
|
"date": "2026-06-23T20:12:19",
|
|
"endpoint": "https://obsidian.example.com",
|
|
"commit": "af16598",
|
|
"echo_workers": 8,
|
|
"echo_timeout": 30,
|
|
"python": "3.10.12",
|
|
"iterations": 6,
|
|
"count_k": 50
|
|
},
|
|
"run_id": "20260623-201219-819b5e",
|
|
"metrics": {
|
|
"resolve": {
|
|
"result": {
|
|
"cases": [
|
|
{
|
|
"mention": "echo",
|
|
"expect": "exact",
|
|
"expect_slug": "echo",
|
|
"resolved_slug": "echo",
|
|
"candidate_slugs": [
|
|
"2026-06-19-other-vault-full-echo-architect",
|
|
"echo",
|
|
"echo-memory-codex-plugin",
|
|
"echo-plugin-build",
|
|
"echo-skill-improvements"
|
|
],
|
|
"verdict": "PASS",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.0,
|
|
"median_ms": 0.0,
|
|
"p90_ms": 0.0,
|
|
"p95_ms": 0.0,
|
|
"max_ms": 0.0,
|
|
"mean_ms": 0.0,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.01,
|
|
"median_ms": 0.1,
|
|
"p90_ms": 0.17,
|
|
"p95_ms": 0.18,
|
|
"max_ms": 0.19,
|
|
"mean_ms": 0.1
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 1.35,
|
|
"median_ms": 1.49,
|
|
"p90_ms": 1.67,
|
|
"p95_ms": 1.68,
|
|
"max_ms": 1.68,
|
|
"mean_ms": 1.51,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 1.82,
|
|
"median_ms": 1.83,
|
|
"p90_ms": 1.84,
|
|
"p95_ms": 1.84,
|
|
"max_ms": 1.85,
|
|
"mean_ms": 1.83
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"mention": "echo memory",
|
|
"expect": "candidates",
|
|
"expect_slug": "echo",
|
|
"resolved_slug": "echo",
|
|
"candidate_slugs": [
|
|
"echo",
|
|
"echo-memory-codex-plugin",
|
|
"echo-plugin-build",
|
|
"echo-skill-improvements",
|
|
"2026-06-19-other-vault-full-echo-architect"
|
|
],
|
|
"verdict": "INFO",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.55,
|
|
"median_ms": 0.56,
|
|
"p90_ms": 0.57,
|
|
"p95_ms": 0.58,
|
|
"max_ms": 0.58,
|
|
"mean_ms": 0.56,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.61,
|
|
"median_ms": 0.61,
|
|
"p90_ms": 0.61,
|
|
"p95_ms": 0.61,
|
|
"max_ms": 0.61,
|
|
"mean_ms": 0.61
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.97,
|
|
"median_ms": 0.99,
|
|
"p90_ms": 1.06,
|
|
"p95_ms": 1.09,
|
|
"max_ms": 1.11,
|
|
"mean_ms": 1.01,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 1.15,
|
|
"median_ms": 1.18,
|
|
"p90_ms": 1.2,
|
|
"p95_ms": 1.21,
|
|
"max_ms": 1.21,
|
|
"mean_ms": 1.18
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"mention": "ECHO plugin",
|
|
"expect": "candidates",
|
|
"expect_slug": "echo",
|
|
"resolved_slug": "echo",
|
|
"candidate_slugs": [
|
|
"echo",
|
|
"echo-memory-codex-plugin",
|
|
"echo-plugin-build",
|
|
"2026-06-19-other-vault-full-echo-architect",
|
|
"example-isp-brand-docs"
|
|
],
|
|
"verdict": "INFO",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.41,
|
|
"median_ms": 0.41,
|
|
"p90_ms": 0.42,
|
|
"p95_ms": 0.42,
|
|
"max_ms": 0.42,
|
|
"mean_ms": 0.41,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.42,
|
|
"median_ms": 0.45,
|
|
"p90_ms": 0.47,
|
|
"p95_ms": 0.48,
|
|
"max_ms": 0.48,
|
|
"mean_ms": 0.45
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.79,
|
|
"median_ms": 0.8,
|
|
"p90_ms": 0.88,
|
|
"p95_ms": 0.9,
|
|
"max_ms": 0.92,
|
|
"mean_ms": 0.83,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.89,
|
|
"median_ms": 0.89,
|
|
"p90_ms": 0.9,
|
|
"p95_ms": 0.9,
|
|
"max_ms": 0.9,
|
|
"mean_ms": 0.89
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"mention": "other-vault",
|
|
"expect": "exact",
|
|
"expect_slug": "other-vault",
|
|
"resolved_slug": "other-vault",
|
|
"candidate_slugs": [
|
|
"2026-06-19-other-vault-full-echo-architect",
|
|
"other-vault"
|
|
],
|
|
"verdict": "PASS",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.0,
|
|
"median_ms": 0.0,
|
|
"p90_ms": 0.0,
|
|
"p95_ms": 0.0,
|
|
"max_ms": 0.0,
|
|
"mean_ms": 0.0,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.0,
|
|
"median_ms": 0.0,
|
|
"p90_ms": 0.0,
|
|
"p95_ms": 0.0,
|
|
"max_ms": 0.0,
|
|
"mean_ms": 0.0
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.73,
|
|
"median_ms": 0.75,
|
|
"p90_ms": 0.85,
|
|
"p95_ms": 0.88,
|
|
"max_ms": 0.91,
|
|
"mean_ms": 0.78,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.8,
|
|
"median_ms": 0.81,
|
|
"p90_ms": 0.82,
|
|
"p95_ms": 0.82,
|
|
"max_ms": 0.82,
|
|
"mean_ms": 0.81
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"mention": "example subject",
|
|
"expect": "exact",
|
|
"expect_slug": "example-subject",
|
|
"resolved_slug": "example-subject",
|
|
"candidate_slugs": [
|
|
"example-subject",
|
|
"operator-mcp-gateway"
|
|
],
|
|
"verdict": "PASS",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.0,
|
|
"median_ms": 0.0,
|
|
"p90_ms": 0.0,
|
|
"p95_ms": 0.0,
|
|
"max_ms": 0.0,
|
|
"mean_ms": 0.0,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.0,
|
|
"median_ms": 0.0,
|
|
"p90_ms": 0.0,
|
|
"p95_ms": 0.0,
|
|
"max_ms": 0.0,
|
|
"mean_ms": 0.0
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.69,
|
|
"median_ms": 0.72,
|
|
"p90_ms": 0.75,
|
|
"p95_ms": 0.75,
|
|
"max_ms": 0.75,
|
|
"mean_ms": 0.72,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.74,
|
|
"median_ms": 0.74,
|
|
"p90_ms": 0.75,
|
|
"p95_ms": 0.75,
|
|
"max_ms": 0.75,
|
|
"mean_ms": 0.74
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"mention": "operator",
|
|
"expect": "candidates",
|
|
"expect_slug": "example-subject",
|
|
"resolved_slug": "example-subject",
|
|
"candidate_slugs": [
|
|
"operator-mcp-gateway",
|
|
"example-subject"
|
|
],
|
|
"verdict": "INFO",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.61,
|
|
"median_ms": 0.65,
|
|
"p90_ms": 0.66,
|
|
"p95_ms": 0.67,
|
|
"max_ms": 0.67,
|
|
"mean_ms": 0.64,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.66,
|
|
"median_ms": 0.68,
|
|
"p90_ms": 0.69,
|
|
"p95_ms": 0.69,
|
|
"max_ms": 0.69,
|
|
"mean_ms": 0.68
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.63,
|
|
"median_ms": 0.63,
|
|
"p90_ms": 0.65,
|
|
"p95_ms": 0.65,
|
|
"max_ms": 0.66,
|
|
"mean_ms": 0.64,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.68,
|
|
"median_ms": 0.69,
|
|
"p90_ms": 0.7,
|
|
"p95_ms": 0.7,
|
|
"max_ms": 0.7,
|
|
"mean_ms": 0.69
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"mention": "zzqx nonexistent entity",
|
|
"expect": "miss",
|
|
"expect_slug": null,
|
|
"resolved_slug": null,
|
|
"candidate_slugs": [],
|
|
"verdict": "PASS",
|
|
"resolve_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.62,
|
|
"median_ms": 0.63,
|
|
"p90_ms": 0.67,
|
|
"p95_ms": 0.67,
|
|
"max_ms": 0.68,
|
|
"mean_ms": 0.64,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.66,
|
|
"median_ms": 0.67,
|
|
"p90_ms": 0.67,
|
|
"p95_ms": 0.67,
|
|
"max_ms": 0.67,
|
|
"mean_ms": 0.67
|
|
}
|
|
},
|
|
"fuzzy_timing": {
|
|
"n": 6,
|
|
"min_ms": 0.61,
|
|
"median_ms": 0.62,
|
|
"p90_ms": 0.65,
|
|
"p95_ms": 0.65,
|
|
"max_ms": 0.66,
|
|
"mean_ms": 0.62,
|
|
"warmup": {
|
|
"n": 2,
|
|
"min_ms": 0.62,
|
|
"median_ms": 0.63,
|
|
"p90_ms": 0.64,
|
|
"p95_ms": 0.64,
|
|
"max_ms": 0.64,
|
|
"mean_ms": 0.63
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"passes": 4,
|
|
"needs_tuning": 3,
|
|
"note": "verdicts are PASS/INFO only; promote to FAIL once the table is confirmed against the live vault"
|
|
},
|
|
"gates": []
|
|
}
|
|
}
|
|
} |