931 lines
24 KiB
JSON
931 lines
24 KiB
JSON
|
{
|
||
|
|
"env": {
|
||
|
|
"date": "2026-06-23T20:17:09",
|
||
|
|
"endpoint": "https://echoapi.alwisp.com",
|
||
|
|
"commit": "af16598",
|
||
|
|
"echo_workers": 8,
|
||
|
|
"echo_timeout": 30,
|
||
|
|
"python": "3.10.12",
|
||
|
|
"iterations": 4,
|
||
|
|
"count_k": 50
|
||
|
|
},
|
||
|
|
"metrics": {
|
||
|
|
"bulk-append": {
|
||
|
|
"result": {
|
||
|
|
"k": 30,
|
||
|
|
"append_write": {
|
||
|
|
"n": 30,
|
||
|
|
"min_ms": 87.11,
|
||
|
|
"median_ms": 104.08,
|
||
|
|
"p90_ms": 190.63,
|
||
|
|
"p95_ms": 198.89,
|
||
|
|
"max_ms": 241.91,
|
||
|
|
"mean_ms": 117.22
|
||
|
|
},
|
||
|
|
"append_skip": {
|
||
|
|
"n": 30,
|
||
|
|
"min_ms": 47.57,
|
||
|
|
"median_ms": 51.42,
|
||
|
|
"p90_ms": 59.82,
|
||
|
|
"p95_ms": 61.57,
|
||
|
|
"max_ms": 73.49,
|
||
|
|
"mean_ms": 53.22
|
||
|
|
},
|
||
|
|
"skips_confirmed": 30,
|
||
|
|
"all_skipped_second_pass": true
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "all_skipped_second_pass",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1,
|
||
|
|
"why": "second pass of identical lines must be 100% idempotent skips (boolean true coerced to 1)"
|
||
|
|
},
|
||
|
|
"value": true,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"bulk-get": {
|
||
|
|
"result": {
|
||
|
|
"k": 30,
|
||
|
|
"workers": 8,
|
||
|
|
"serial": {
|
||
|
|
"n": 5,
|
||
|
|
"min_ms": 1520.47,
|
||
|
|
"median_ms": 1606.86,
|
||
|
|
"p90_ms": 1805.72,
|
||
|
|
"p95_ms": 1835.83,
|
||
|
|
"max_ms": 1865.94,
|
||
|
|
"mean_ms": 1658.6,
|
||
|
|
"warmup": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 1601.2,
|
||
|
|
"median_ms": 1601.2,
|
||
|
|
"p90_ms": 1601.2,
|
||
|
|
"p95_ms": 1601.2,
|
||
|
|
"max_ms": 1601.2,
|
||
|
|
"mean_ms": 1601.2
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"concurrent": {
|
||
|
|
"n": 5,
|
||
|
|
"min_ms": 340.31,
|
||
|
|
"median_ms": 356.48,
|
||
|
|
"p90_ms": 358.82,
|
||
|
|
"p95_ms": 359.32,
|
||
|
|
"max_ms": 359.83,
|
||
|
|
"mean_ms": 351.42,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 353.43,
|
||
|
|
"median_ms": 434.94,
|
||
|
|
"p90_ms": 500.15,
|
||
|
|
"p95_ms": 508.3,
|
||
|
|
"max_ms": 516.45,
|
||
|
|
"mean_ms": 434.94
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"speedup_ratio": 4.51
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "speedup_ratio",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1.5,
|
||
|
|
"why": "controlled-K concurrent vs serial; lower bar than full-vault since K is small"
|
||
|
|
},
|
||
|
|
"value": 4.51,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"bulk-put": {
|
||
|
|
"result": {
|
||
|
|
"k": 30,
|
||
|
|
"per_put": {
|
||
|
|
"n": 30,
|
||
|
|
"min_ms": 100.09,
|
||
|
|
"median_ms": 114.98,
|
||
|
|
"p90_ms": 212.46,
|
||
|
|
"p95_ms": 236.18,
|
||
|
|
"max_ms": 330.77,
|
||
|
|
"mean_ms": 143.25
|
||
|
|
},
|
||
|
|
"total_ms": 4297.5,
|
||
|
|
"puts_per_sec": 7.0
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
},
|
||
|
|
"cache": {
|
||
|
|
"result": {
|
||
|
|
"requested": 60,
|
||
|
|
"unique": 20,
|
||
|
|
"returned": 20,
|
||
|
|
"deduped": true,
|
||
|
|
"elapsed_ms": 307.13
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "deduped",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1,
|
||
|
|
"why": "read_many must collapse a 3x-duplicated path list to the unique set"
|
||
|
|
},
|
||
|
|
"value": true,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"expand-graph": {
|
||
|
|
"result": {
|
||
|
|
"subjects": [
|
||
|
|
{
|
||
|
|
"label": "hub-echo",
|
||
|
|
"query": "echo",
|
||
|
|
"neighbours": 120,
|
||
|
|
"serial_ms": 2227.67,
|
||
|
|
"concurrent_ms": 642.2,
|
||
|
|
"speedup": 3.47,
|
||
|
|
"ranking_identical": true,
|
||
|
|
"scores_identical": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"label": "hub-prefs",
|
||
|
|
"query": "operator preferences",
|
||
|
|
"neighbours": 126,
|
||
|
|
"serial_ms": 2910.46,
|
||
|
|
"concurrent_ms": 698.8,
|
||
|
|
"speedup": 4.16,
|
||
|
|
"ranking_identical": true,
|
||
|
|
"scores_identical": true
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"min_speedup": 3.47,
|
||
|
|
"all_ranking_identical": true,
|
||
|
|
"all_scores_identical": true
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "min_speedup",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 2.0,
|
||
|
|
"why": "recall()'s graph layer must fetch each BFS hop concurrently (read_many), not serially per node; <2x means expand_graph regressed to the pre-fix serial walk"
|
||
|
|
},
|
||
|
|
"value": 3.47,
|
||
|
|
"pass": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "all_ranking_identical",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1,
|
||
|
|
"why": "the concurrent expansion must return the same ranked neighbours as the serial reference"
|
||
|
|
},
|
||
|
|
"value": true,
|
||
|
|
"pass": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "all_scores_identical",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1,
|
||
|
|
"why": "decayed scores must match the serial reference to within float tolerance"
|
||
|
|
},
|
||
|
|
"value": true,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"index": {
|
||
|
|
"result": {
|
||
|
|
"script": "sweep.py",
|
||
|
|
"elapsed_ms": 4629.0,
|
||
|
|
"exit_code": 0,
|
||
|
|
"ok": true,
|
||
|
|
"stderr_tail": []
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
},
|
||
|
|
"lint": {
|
||
|
|
"result": {
|
||
|
|
"script": "vault_lint.py",
|
||
|
|
"elapsed_ms": 5126.4,
|
||
|
|
"exit_code": 1,
|
||
|
|
"ok": true,
|
||
|
|
"stderr_tail": []
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
},
|
||
|
|
"lock": {
|
||
|
|
"result": {
|
||
|
|
"acquire_free_ms": 159.18,
|
||
|
|
"acquire_free_rc": 0,
|
||
|
|
"contended_ms": 57.89,
|
||
|
|
"contended_rc_expected_75": 75,
|
||
|
|
"release_ms": 132.92,
|
||
|
|
"ok": true
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "ok",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1,
|
||
|
|
"why": "free acquire returns 0 and a contended acquire fast-fails with exit 75"
|
||
|
|
},
|
||
|
|
"value": true,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"pool-warmup": {
|
||
|
|
"result": {
|
||
|
|
"probe": "_agent/echo-vault.md",
|
||
|
|
"cold_ms": 163.25,
|
||
|
|
"warm": {
|
||
|
|
"n": 12,
|
||
|
|
"min_ms": 45.64,
|
||
|
|
"median_ms": 50.29,
|
||
|
|
"p90_ms": 52.51,
|
||
|
|
"p95_ms": 53.79,
|
||
|
|
"max_ms": 55.28,
|
||
|
|
"mean_ms": 50.26,
|
||
|
|
"warmup": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 51.68,
|
||
|
|
"median_ms": 51.68,
|
||
|
|
"p90_ms": 51.68,
|
||
|
|
"p95_ms": 51.68,
|
||
|
|
"max_ms": 51.68,
|
||
|
|
"mean_ms": 51.68
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"cold_over_warm_ratio": 3.25
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "cold_over_warm_ratio",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1.5,
|
||
|
|
"why": "cold request must be meaningfully slower than warm \u2014 proves keep-alive is reusing the connection; a ratio near 1.0 means every request is re-handshaking (the pre-1.1.0 bug)"
|
||
|
|
},
|
||
|
|
"value": 3.25,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"queue": {
|
||
|
|
"result": {
|
||
|
|
"k": 10,
|
||
|
|
"enqueue": {
|
||
|
|
"n": 10,
|
||
|
|
"min_ms": 0.07,
|
||
|
|
"median_ms": 0.08,
|
||
|
|
"p90_ms": 0.18,
|
||
|
|
"p95_ms": 0.2,
|
||
|
|
"max_ms": 0.22,
|
||
|
|
"mean_ms": 0.1
|
||
|
|
},
|
||
|
|
"pending_after_enqueue": 10,
|
||
|
|
"note": "flush replay intentionally not run against the live vault; enqueue path timed only. See README queue caveat."
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
},
|
||
|
|
"read-full": {
|
||
|
|
"result": {
|
||
|
|
"note_count": 197,
|
||
|
|
"workers": 8,
|
||
|
|
"serial": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 11376.08,
|
||
|
|
"median_ms": 11376.08,
|
||
|
|
"p90_ms": 11376.08,
|
||
|
|
"p95_ms": 11376.08,
|
||
|
|
"max_ms": 11376.08,
|
||
|
|
"mean_ms": 11376.08
|
||
|
|
},
|
||
|
|
"concurrent": {
|
||
|
|
"n": 4,
|
||
|
|
"min_ms": 1559.79,
|
||
|
|
"median_ms": 1595.31,
|
||
|
|
"p90_ms": 1740.92,
|
||
|
|
"p95_ms": 1768.05,
|
||
|
|
"max_ms": 1795.17,
|
||
|
|
"mean_ms": 1636.4,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 1641.35,
|
||
|
|
"median_ms": 1652.91,
|
||
|
|
"p90_ms": 1662.17,
|
||
|
|
"p95_ms": 1663.32,
|
||
|
|
"max_ms": 1664.48,
|
||
|
|
"mean_ms": 1652.91
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"speedup_ratio": 7.13,
|
||
|
|
"notes_per_sec_concurrent": 123.5
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "speedup_ratio",
|
||
|
|
"op": ">=",
|
||
|
|
"min": 1.8,
|
||
|
|
"why": "concurrent read_many must beat serial by the 1.1.0 concurrency margin; <1.8 means concurrency regressed"
|
||
|
|
},
|
||
|
|
"value": 7.13,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"resolve": {
|
||
|
|
"result": {
|
||
|
|
"cases": [
|
||
|
|
{
|
||
|
|
"mention": "echo",
|
||
|
|
"expect": "exact",
|
||
|
|
"expect_slug": "echo",
|
||
|
|
"resolved_slug": "echo",
|
||
|
|
"candidate_slugs": [
|
||
|
|
"2026-06-19-goldbrain-full-echo-architect",
|
||
|
|
"echo",
|
||
|
|
"echo-memory-codex-plugin",
|
||
|
|
"echo-plugin-build",
|
||
|
|
"echo-skill-improvements"
|
||
|
|
],
|
||
|
|
"verdict": "PASS",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.0,
|
||
|
|
"median_ms": 0.0,
|
||
|
|
"p90_ms": 0.0,
|
||
|
|
"p95_ms": 0.0,
|
||
|
|
"max_ms": 0.0,
|
||
|
|
"mean_ms": 0.0,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.01,
|
||
|
|
"median_ms": 0.1,
|
||
|
|
"p90_ms": 0.17,
|
||
|
|
"p95_ms": 0.18,
|
||
|
|
"max_ms": 0.19,
|
||
|
|
"mean_ms": 0.1
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 1.35,
|
||
|
|
"median_ms": 1.49,
|
||
|
|
"p90_ms": 1.67,
|
||
|
|
"p95_ms": 1.68,
|
||
|
|
"max_ms": 1.68,
|
||
|
|
"mean_ms": 1.51,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 1.82,
|
||
|
|
"median_ms": 1.83,
|
||
|
|
"p90_ms": 1.84,
|
||
|
|
"p95_ms": 1.84,
|
||
|
|
"max_ms": 1.85,
|
||
|
|
"mean_ms": 1.83
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mention": "echo memory",
|
||
|
|
"expect": "candidates",
|
||
|
|
"expect_slug": "echo",
|
||
|
|
"resolved_slug": "echo",
|
||
|
|
"candidate_slugs": [
|
||
|
|
"echo",
|
||
|
|
"echo-memory-codex-plugin",
|
||
|
|
"echo-plugin-build",
|
||
|
|
"echo-skill-improvements",
|
||
|
|
"2026-06-19-goldbrain-full-echo-architect"
|
||
|
|
],
|
||
|
|
"verdict": "INFO",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.55,
|
||
|
|
"median_ms": 0.56,
|
||
|
|
"p90_ms": 0.57,
|
||
|
|
"p95_ms": 0.58,
|
||
|
|
"max_ms": 0.58,
|
||
|
|
"mean_ms": 0.56,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.61,
|
||
|
|
"median_ms": 0.61,
|
||
|
|
"p90_ms": 0.61,
|
||
|
|
"p95_ms": 0.61,
|
||
|
|
"max_ms": 0.61,
|
||
|
|
"mean_ms": 0.61
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.97,
|
||
|
|
"median_ms": 0.99,
|
||
|
|
"p90_ms": 1.06,
|
||
|
|
"p95_ms": 1.09,
|
||
|
|
"max_ms": 1.11,
|
||
|
|
"mean_ms": 1.01,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 1.15,
|
||
|
|
"median_ms": 1.18,
|
||
|
|
"p90_ms": 1.2,
|
||
|
|
"p95_ms": 1.21,
|
||
|
|
"max_ms": 1.21,
|
||
|
|
"mean_ms": 1.18
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mention": "ECHO plugin",
|
||
|
|
"expect": "candidates",
|
||
|
|
"expect_slug": "echo",
|
||
|
|
"resolved_slug": "echo",
|
||
|
|
"candidate_slugs": [
|
||
|
|
"echo",
|
||
|
|
"echo-memory-codex-plugin",
|
||
|
|
"echo-plugin-build",
|
||
|
|
"2026-06-19-goldbrain-full-echo-architect",
|
||
|
|
"alabama-wisp-brand-docs"
|
||
|
|
],
|
||
|
|
"verdict": "INFO",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.41,
|
||
|
|
"median_ms": 0.41,
|
||
|
|
"p90_ms": 0.42,
|
||
|
|
"p95_ms": 0.42,
|
||
|
|
"max_ms": 0.42,
|
||
|
|
"mean_ms": 0.41,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.42,
|
||
|
|
"median_ms": 0.45,
|
||
|
|
"p90_ms": 0.47,
|
||
|
|
"p95_ms": 0.48,
|
||
|
|
"max_ms": 0.48,
|
||
|
|
"mean_ms": 0.45
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.79,
|
||
|
|
"median_ms": 0.8,
|
||
|
|
"p90_ms": 0.88,
|
||
|
|
"p95_ms": 0.9,
|
||
|
|
"max_ms": 0.92,
|
||
|
|
"mean_ms": 0.83,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.89,
|
||
|
|
"median_ms": 0.89,
|
||
|
|
"p90_ms": 0.9,
|
||
|
|
"p95_ms": 0.9,
|
||
|
|
"max_ms": 0.9,
|
||
|
|
"mean_ms": 0.89
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mention": "goldbrain",
|
||
|
|
"expect": "exact",
|
||
|
|
"expect_slug": "goldbrain",
|
||
|
|
"resolved_slug": "goldbrain",
|
||
|
|
"candidate_slugs": [
|
||
|
|
"2026-06-19-goldbrain-full-echo-architect",
|
||
|
|
"goldbrain"
|
||
|
|
],
|
||
|
|
"verdict": "PASS",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.0,
|
||
|
|
"median_ms": 0.0,
|
||
|
|
"p90_ms": 0.0,
|
||
|
|
"p95_ms": 0.0,
|
||
|
|
"max_ms": 0.0,
|
||
|
|
"mean_ms": 0.0,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.0,
|
||
|
|
"median_ms": 0.0,
|
||
|
|
"p90_ms": 0.0,
|
||
|
|
"p95_ms": 0.0,
|
||
|
|
"max_ms": 0.0,
|
||
|
|
"mean_ms": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.73,
|
||
|
|
"median_ms": 0.75,
|
||
|
|
"p90_ms": 0.85,
|
||
|
|
"p95_ms": 0.88,
|
||
|
|
"max_ms": 0.91,
|
||
|
|
"mean_ms": 0.78,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.8,
|
||
|
|
"median_ms": 0.81,
|
||
|
|
"p90_ms": 0.82,
|
||
|
|
"p95_ms": 0.82,
|
||
|
|
"max_ms": 0.82,
|
||
|
|
"mean_ms": 0.81
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mention": "jason stedwell",
|
||
|
|
"expect": "exact",
|
||
|
|
"expect_slug": "jason-stedwell",
|
||
|
|
"resolved_slug": "jason-stedwell",
|
||
|
|
"candidate_slugs": [
|
||
|
|
"jason-stedwell",
|
||
|
|
"jason-mcp-gateway"
|
||
|
|
],
|
||
|
|
"verdict": "PASS",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.0,
|
||
|
|
"median_ms": 0.0,
|
||
|
|
"p90_ms": 0.0,
|
||
|
|
"p95_ms": 0.0,
|
||
|
|
"max_ms": 0.0,
|
||
|
|
"mean_ms": 0.0,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.0,
|
||
|
|
"median_ms": 0.0,
|
||
|
|
"p90_ms": 0.0,
|
||
|
|
"p95_ms": 0.0,
|
||
|
|
"max_ms": 0.0,
|
||
|
|
"mean_ms": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.69,
|
||
|
|
"median_ms": 0.72,
|
||
|
|
"p90_ms": 0.75,
|
||
|
|
"p95_ms": 0.75,
|
||
|
|
"max_ms": 0.75,
|
||
|
|
"mean_ms": 0.72,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.74,
|
||
|
|
"median_ms": 0.74,
|
||
|
|
"p90_ms": 0.75,
|
||
|
|
"p95_ms": 0.75,
|
||
|
|
"max_ms": 0.75,
|
||
|
|
"mean_ms": 0.74
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mention": "jason",
|
||
|
|
"expect": "candidates",
|
||
|
|
"expect_slug": "jason-stedwell",
|
||
|
|
"resolved_slug": "jason-stedwell",
|
||
|
|
"candidate_slugs": [
|
||
|
|
"jason-mcp-gateway",
|
||
|
|
"jason-stedwell"
|
||
|
|
],
|
||
|
|
"verdict": "INFO",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.61,
|
||
|
|
"median_ms": 0.65,
|
||
|
|
"p90_ms": 0.66,
|
||
|
|
"p95_ms": 0.67,
|
||
|
|
"max_ms": 0.67,
|
||
|
|
"mean_ms": 0.64,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.66,
|
||
|
|
"median_ms": 0.68,
|
||
|
|
"p90_ms": 0.69,
|
||
|
|
"p95_ms": 0.69,
|
||
|
|
"max_ms": 0.69,
|
||
|
|
"mean_ms": 0.68
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.63,
|
||
|
|
"median_ms": 0.63,
|
||
|
|
"p90_ms": 0.65,
|
||
|
|
"p95_ms": 0.65,
|
||
|
|
"max_ms": 0.66,
|
||
|
|
"mean_ms": 0.64,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.68,
|
||
|
|
"median_ms": 0.69,
|
||
|
|
"p90_ms": 0.7,
|
||
|
|
"p95_ms": 0.7,
|
||
|
|
"max_ms": 0.7,
|
||
|
|
"mean_ms": 0.69
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"mention": "zzqx nonexistent entity",
|
||
|
|
"expect": "miss",
|
||
|
|
"expect_slug": null,
|
||
|
|
"resolved_slug": null,
|
||
|
|
"candidate_slugs": [],
|
||
|
|
"verdict": "PASS",
|
||
|
|
"resolve_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.62,
|
||
|
|
"median_ms": 0.63,
|
||
|
|
"p90_ms": 0.67,
|
||
|
|
"p95_ms": 0.67,
|
||
|
|
"max_ms": 0.68,
|
||
|
|
"mean_ms": 0.64,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.66,
|
||
|
|
"median_ms": 0.67,
|
||
|
|
"p90_ms": 0.67,
|
||
|
|
"p95_ms": 0.67,
|
||
|
|
"max_ms": 0.67,
|
||
|
|
"mean_ms": 0.67
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"fuzzy_timing": {
|
||
|
|
"n": 6,
|
||
|
|
"min_ms": 0.61,
|
||
|
|
"median_ms": 0.62,
|
||
|
|
"p90_ms": 0.65,
|
||
|
|
"p95_ms": 0.65,
|
||
|
|
"max_ms": 0.66,
|
||
|
|
"mean_ms": 0.62,
|
||
|
|
"warmup": {
|
||
|
|
"n": 2,
|
||
|
|
"min_ms": 0.62,
|
||
|
|
"median_ms": 0.63,
|
||
|
|
"p90_ms": 0.64,
|
||
|
|
"p95_ms": 0.64,
|
||
|
|
"max_ms": 0.64,
|
||
|
|
"mean_ms": 0.63
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"passes": 4,
|
||
|
|
"needs_tuning": 3,
|
||
|
|
"note": "verdicts are PASS/INFO only; promote to FAIL once the table is confirmed against the live vault"
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
},
|
||
|
|
"soak": {
|
||
|
|
"result": {
|
||
|
|
"seconds": 22,
|
||
|
|
"iterations": 15,
|
||
|
|
"errors": 0,
|
||
|
|
"per_pass": {
|
||
|
|
"n": 15,
|
||
|
|
"min_ms": 1472.88,
|
||
|
|
"median_ms": 1551.67,
|
||
|
|
"p90_ms": 1648.45,
|
||
|
|
"p95_ms": 1702.66,
|
||
|
|
"max_ms": 1820.58,
|
||
|
|
"mean_ms": 1569.81
|
||
|
|
},
|
||
|
|
"early_mean_ms": 1575.63,
|
||
|
|
"late_mean_ms": 1564.72,
|
||
|
|
"drift_ratio_late_over_early": 0.99
|
||
|
|
},
|
||
|
|
"gates": [
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "errors",
|
||
|
|
"op": "<=",
|
||
|
|
"min": 0,
|
||
|
|
"why": "no read errors across the soak window \u2014 connection leaks/pool exhaustion would surface here"
|
||
|
|
},
|
||
|
|
"value": 0,
|
||
|
|
"pass": true
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"rule": {
|
||
|
|
"field": "drift_ratio_late_over_early",
|
||
|
|
"op": "<=",
|
||
|
|
"min": 1.5,
|
||
|
|
"why": "late passes must not be much slower than early ones; >1.5 suggests a leak or degrading pool"
|
||
|
|
},
|
||
|
|
"value": 0.99,
|
||
|
|
"pass": true
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"subject-pull": {
|
||
|
|
"result": {
|
||
|
|
"subjects": [
|
||
|
|
{
|
||
|
|
"label": "apta",
|
||
|
|
"query": "APTA",
|
||
|
|
"resolve_floor": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 0.75,
|
||
|
|
"median_ms": 0.75,
|
||
|
|
"p90_ms": 0.75,
|
||
|
|
"p95_ms": 0.75,
|
||
|
|
"max_ms": 0.75,
|
||
|
|
"mean_ms": 0.75
|
||
|
|
},
|
||
|
|
"search_only": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 61.87,
|
||
|
|
"median_ms": 61.87,
|
||
|
|
"p90_ms": 61.87,
|
||
|
|
"p95_ms": 61.87,
|
||
|
|
"max_ms": 61.87,
|
||
|
|
"mean_ms": 61.87
|
||
|
|
},
|
||
|
|
"recall_total": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2231.31,
|
||
|
|
"median_ms": 2231.31,
|
||
|
|
"p90_ms": 2231.31,
|
||
|
|
"p95_ms": 2231.31,
|
||
|
|
"max_ms": 2231.31,
|
||
|
|
"mean_ms": 2231.31
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"label": "capmetro",
|
||
|
|
"query": "CapMetro",
|
||
|
|
"resolve_floor": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 0.03,
|
||
|
|
"median_ms": 0.03,
|
||
|
|
"p90_ms": 0.03,
|
||
|
|
"p95_ms": 0.03,
|
||
|
|
"max_ms": 0.03,
|
||
|
|
"mean_ms": 0.03
|
||
|
|
},
|
||
|
|
"search_only": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 57.73,
|
||
|
|
"median_ms": 57.73,
|
||
|
|
"p90_ms": 57.73,
|
||
|
|
"p95_ms": 57.73,
|
||
|
|
"max_ms": 57.73,
|
||
|
|
"mean_ms": 57.73
|
||
|
|
},
|
||
|
|
"recall_total": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2116.06,
|
||
|
|
"median_ms": 2116.06,
|
||
|
|
"p90_ms": 2116.06,
|
||
|
|
"p95_ms": 2116.06,
|
||
|
|
"max_ms": 2116.06,
|
||
|
|
"mean_ms": 2116.06
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"label": "hub-echo",
|
||
|
|
"query": "echo",
|
||
|
|
"resolve_floor": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 0.03,
|
||
|
|
"median_ms": 0.03,
|
||
|
|
"p90_ms": 0.03,
|
||
|
|
"p95_ms": 0.03,
|
||
|
|
"max_ms": 0.03,
|
||
|
|
"mean_ms": 0.03
|
||
|
|
},
|
||
|
|
"search_only": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 70.56,
|
||
|
|
"median_ms": 70.56,
|
||
|
|
"p90_ms": 70.56,
|
||
|
|
"p95_ms": 70.56,
|
||
|
|
"max_ms": 70.56,
|
||
|
|
"mean_ms": 70.56
|
||
|
|
},
|
||
|
|
"recall_total": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2403.62,
|
||
|
|
"median_ms": 2403.62,
|
||
|
|
"p90_ms": 2403.62,
|
||
|
|
"p95_ms": 2403.62,
|
||
|
|
"max_ms": 2403.62,
|
||
|
|
"mean_ms": 2403.62
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"label": "hub-prefs",
|
||
|
|
"query": "operator preferences",
|
||
|
|
"resolve_floor": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 0.04,
|
||
|
|
"median_ms": 0.04,
|
||
|
|
"p90_ms": 0.04,
|
||
|
|
"p95_ms": 0.04,
|
||
|
|
"max_ms": 0.04,
|
||
|
|
"mean_ms": 0.04
|
||
|
|
},
|
||
|
|
"search_only": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2011.13,
|
||
|
|
"median_ms": 2011.13,
|
||
|
|
"p90_ms": 2011.13,
|
||
|
|
"p95_ms": 2011.13,
|
||
|
|
"max_ms": 2011.13,
|
||
|
|
"mean_ms": 2011.13
|
||
|
|
},
|
||
|
|
"recall_total": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2994.84,
|
||
|
|
"median_ms": 2994.84,
|
||
|
|
"p90_ms": 2994.84,
|
||
|
|
"p95_ms": 2994.84,
|
||
|
|
"max_ms": 2994.84,
|
||
|
|
"mean_ms": 2994.84
|
||
|
|
}
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"label": "leaf-rivnut",
|
||
|
|
"query": "rivnut torque spec",
|
||
|
|
"resolve_floor": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2.15,
|
||
|
|
"median_ms": 2.15,
|
||
|
|
"p90_ms": 2.15,
|
||
|
|
"p95_ms": 2.15,
|
||
|
|
"max_ms": 2.15,
|
||
|
|
"mean_ms": 2.15
|
||
|
|
},
|
||
|
|
"search_only": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2013.58,
|
||
|
|
"median_ms": 2013.58,
|
||
|
|
"p90_ms": 2013.58,
|
||
|
|
"p95_ms": 2013.58,
|
||
|
|
"max_ms": 2013.58,
|
||
|
|
"mean_ms": 2013.58
|
||
|
|
},
|
||
|
|
"recall_total": {
|
||
|
|
"n": 1,
|
||
|
|
"min_ms": 2371.32,
|
||
|
|
"median_ms": 2371.32,
|
||
|
|
"p90_ms": 2371.32,
|
||
|
|
"p95_ms": 2371.32,
|
||
|
|
"max_ms": 2371.32,
|
||
|
|
"mean_ms": 2371.32
|
||
|
|
}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
},
|
||
|
|
"worker-sweep": {
|
||
|
|
"result": {
|
||
|
|
"note_count": 197,
|
||
|
|
"sweep": [
|
||
|
|
{
|
||
|
|
"workers": 1,
|
||
|
|
"median_ms": 11917.49,
|
||
|
|
"p95_ms": 11917.49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"workers": 2,
|
||
|
|
"median_ms": 5740.0,
|
||
|
|
"p95_ms": 5740.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"workers": 4,
|
||
|
|
"median_ms": 3034.3,
|
||
|
|
"p95_ms": 3034.3
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"workers": 8,
|
||
|
|
"median_ms": 1583.83,
|
||
|
|
"p95_ms": 1583.83
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"workers": 16,
|
||
|
|
"median_ms": 1277.77,
|
||
|
|
"p95_ms": 1277.77
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"knee_workers": 16,
|
||
|
|
"default_workers": 8
|
||
|
|
},
|
||
|
|
"gates": []
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|