200 lines
No EOL
4 KiB
JSON
200 lines
No EOL
4 KiB
JSON
{
|
|
"dataset_version": "2026-08-19.v1",
|
|
"index": {
|
|
"kind": "inverted_keyword_v1",
|
|
"tools": 180,
|
|
"postings": 528,
|
|
"build_ms": 18.359,
|
|
"memory_bytes_est": 121248
|
|
},
|
|
"five_metadata_misses": [
|
|
{
|
|
"id": "ppl-05",
|
|
"query": "مانده حساب علی چقدر است",
|
|
"expected": [
|
|
"get_person_balance"
|
|
],
|
|
"hit": true,
|
|
"rank": 1
|
|
},
|
|
{
|
|
"id": "acc-21",
|
|
"query": "لیست ارزها",
|
|
"expected": [
|
|
"list_currencies"
|
|
],
|
|
"hit": true,
|
|
"rank": 1
|
|
},
|
|
{
|
|
"id": "hard-gen-01",
|
|
"query": "یک گزارش کلی از اوضاع بده",
|
|
"expected": [
|
|
"get_business_dashboard"
|
|
],
|
|
"hit": true,
|
|
"rank": 1
|
|
},
|
|
{
|
|
"id": "plat-01",
|
|
"query": "اطلاعات این کسبوکار",
|
|
"expected": [
|
|
"get_business_info"
|
|
],
|
|
"hit": true,
|
|
"rank": 1
|
|
},
|
|
{
|
|
"id": "exp-158",
|
|
"query": "اطلاعات کسبوکار نشست",
|
|
"expected": [
|
|
"get_business_info"
|
|
],
|
|
"hit": true,
|
|
"rank": 1
|
|
}
|
|
],
|
|
"candidate_recall": [
|
|
{
|
|
"n": 20,
|
|
"queries": 443,
|
|
"candidate_recall": 0.9549,
|
|
"latency_p50_ms": 0.762,
|
|
"latency_p95_ms": 1.01,
|
|
"avg_latency_ms": 0.774
|
|
},
|
|
{
|
|
"n": 50,
|
|
"queries": 443,
|
|
"candidate_recall": 0.9955,
|
|
"latency_p50_ms": 0.849,
|
|
"latency_p95_ms": 1.157,
|
|
"avg_latency_ms": 0.828
|
|
},
|
|
{
|
|
"n": 100,
|
|
"queries": 443,
|
|
"candidate_recall": 1.0,
|
|
"latency_p50_ms": 0.858,
|
|
"latency_p95_ms": 1.34,
|
|
"avg_latency_ms": 0.879
|
|
},
|
|
{
|
|
"n": 200,
|
|
"queries": 443,
|
|
"candidate_recall": 1.0,
|
|
"latency_p50_ms": 0.852,
|
|
"latency_p95_ms": 1.325,
|
|
"avg_latency_ms": 0.87
|
|
}
|
|
],
|
|
"remaining_misses_at_100": [],
|
|
"target_candidate_recall": 0.99,
|
|
"candidate_recall_100_met": true,
|
|
"indexed_final": [
|
|
{
|
|
"k": 10,
|
|
"candidate_n": 100,
|
|
"recall": 0.9616,
|
|
"mrr": 0.7931,
|
|
"ndcg": 0.8091
|
|
},
|
|
{
|
|
"k": 15,
|
|
"candidate_n": 100,
|
|
"recall": 0.9819,
|
|
"mrr": 0.7945,
|
|
"ndcg": 0.8166
|
|
},
|
|
{
|
|
"k": 20,
|
|
"candidate_n": 100,
|
|
"recall": 0.9887,
|
|
"mrr": 0.7948,
|
|
"ndcg": 0.8191
|
|
},
|
|
{
|
|
"k": 48,
|
|
"candidate_n": 100,
|
|
"recall": 0.9887,
|
|
"mrr": 0.7947,
|
|
"ndcg": 0.8233
|
|
}
|
|
],
|
|
"baseline_final": [
|
|
{
|
|
"k": 10,
|
|
"recall": 0.9616
|
|
},
|
|
{
|
|
"k": 15,
|
|
"recall": 0.9819
|
|
},
|
|
{
|
|
"k": 20,
|
|
"recall": 0.9887
|
|
},
|
|
{
|
|
"k": 48,
|
|
"recall": 0.9887
|
|
}
|
|
],
|
|
"phase6_baseline": {
|
|
"10": 0.9549,
|
|
"15": 0.9752,
|
|
"20": 0.9819,
|
|
"48": 0.9819
|
|
},
|
|
"no_match": {
|
|
"queries": 30,
|
|
"empty_candidate_set": 15,
|
|
"empty_rate": 0.5
|
|
},
|
|
"incremental_consistency": {
|
|
"pass": true,
|
|
"tools": 6,
|
|
"posting_keys": 65
|
|
},
|
|
"latency": [
|
|
{
|
|
"size": 180,
|
|
"control_p50_ms": 7.807,
|
|
"control_p95_ms": 8.074,
|
|
"indexed_p50_ms": 3.368,
|
|
"indexed_p95_ms": 3.459
|
|
},
|
|
{
|
|
"size": 1200,
|
|
"control_p50_ms": 18.553,
|
|
"control_p95_ms": 18.764,
|
|
"indexed_p50_ms": 3.886,
|
|
"indexed_p95_ms": 3.947
|
|
},
|
|
{
|
|
"size": 5000,
|
|
"control_p50_ms": 58.849,
|
|
"control_p95_ms": 61.097,
|
|
"indexed_p50_ms": 4.624,
|
|
"indexed_p95_ms": 4.762
|
|
},
|
|
{
|
|
"size": 10000,
|
|
"control_p50_ms": 110.76,
|
|
"control_p95_ms": 112.486,
|
|
"indexed_p50_ms": 6.129,
|
|
"indexed_p95_ms": 6.628
|
|
}
|
|
],
|
|
"flags": {
|
|
"INDEXED_CANDIDATE_RETRIEVAL": false,
|
|
"INDEXED_CANDIDATE_ROLLOUT_PERCENT": 0,
|
|
"INDEXED_CANDIDATE_FALLBACK": true,
|
|
"VECTOR_CANDIDATE_RETRIEVAL": false,
|
|
"CANDIDATE_RETRIEVAL_N": 100,
|
|
"HYBRID_TOOL_DISCOVERY": false,
|
|
"ADAPTIVE_DISCOVERY_K": false,
|
|
"ADAPTIVE_K_ROLLOUT_PERCENT": 0,
|
|
"PROGRESSIVE_SCHEMA_LOADING": false
|
|
},
|
|
"note": "Indexed retrieval is canary-ready at 0%. Production still ranks the full authorized set. Adaptive K HOLD. Hybrid OFF."
|
|
} |