forked from hesabix/arc
915 lines
No EOL
23 KiB
JSON
915 lines
No EOL
23 KiB
JSON
{
|
|
"dataset_version": "2026-08-19.v1",
|
|
"tool_catalog_version": "180.phase4",
|
|
"dataset": {
|
|
"dataset_version": "2026-08-19.v1",
|
|
"tool_catalog_version": "180.phase4",
|
|
"queries": 473,
|
|
"match_queries": 443,
|
|
"no_match_queries": 30,
|
|
"tools_covered": 175,
|
|
"capabilities_covered": 48,
|
|
"categories": [
|
|
"CRM",
|
|
"accounting",
|
|
"customer",
|
|
"inventory",
|
|
"invoice",
|
|
"legacy",
|
|
"memory",
|
|
"none",
|
|
"purchase",
|
|
"reports",
|
|
"sales",
|
|
"support",
|
|
"wallet",
|
|
"workflow"
|
|
],
|
|
"write_queries": 69,
|
|
"destructive_queries": 20,
|
|
"report_queries": 60,
|
|
"memory_queries": 13,
|
|
"hard_queries": 112,
|
|
"legacy_queries": 22
|
|
},
|
|
"baseline_k48": {
|
|
"k": 48,
|
|
"recall": 0.9819,
|
|
"mrr": 0.7871,
|
|
"precision": 0.1209,
|
|
"ndcg": 0.818,
|
|
"retrieval_success_rate": 0.9819
|
|
},
|
|
"legacy_k48": {
|
|
"k": 48,
|
|
"recall": 1.0,
|
|
"queries": 22
|
|
},
|
|
"by_k": {
|
|
"5": {
|
|
"k": 5,
|
|
"recall": 0.8962,
|
|
"multi_tool_recall": 0.8,
|
|
"mrr": 0.7767,
|
|
"precision": 0.3514,
|
|
"ndcg": 0.7788,
|
|
"retrieval_success_rate": 0.8939,
|
|
"avg_schema_tokens": 1380,
|
|
"context_share": 0.0493,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.7931,
|
|
"accounting": 0.908,
|
|
"customer": 0.8824,
|
|
"inventory": 0.8167,
|
|
"invoice": 1.0,
|
|
"legacy": 0.9545,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9211,
|
|
"sales": 1.0,
|
|
"support": 0.7593,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 0.9478,
|
|
"hard": 0.8539,
|
|
"medium": 0.8731
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9661,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 0.8966
|
|
}
|
|
},
|
|
"10": {
|
|
"k": 10,
|
|
"recall": 0.9549,
|
|
"multi_tool_recall": 0.8,
|
|
"mrr": 0.7856,
|
|
"precision": 0.2164,
|
|
"ndcg": 0.8042,
|
|
"retrieval_success_rate": 0.9526,
|
|
"avg_schema_tokens": 2605,
|
|
"context_share": 0.093,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 0.954,
|
|
"customer": 1.0,
|
|
"inventory": 0.9,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.8889,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 0.9776,
|
|
"hard": 0.9438,
|
|
"medium": 0.9442
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"15": {
|
|
"k": 15,
|
|
"recall": 0.9752,
|
|
"multi_tool_recall": 0.9,
|
|
"mrr": 0.787,
|
|
"precision": 0.1706,
|
|
"ndcg": 0.8113,
|
|
"retrieval_success_rate": 0.9752,
|
|
"avg_schema_tokens": 3706,
|
|
"context_share": 0.1324,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 0.9655,
|
|
"customer": 1.0,
|
|
"inventory": 1.0,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9259,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 0.9925,
|
|
"hard": 0.9551,
|
|
"medium": 0.9746
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"20": {
|
|
"k": 20,
|
|
"recall": 0.9819,
|
|
"multi_tool_recall": 0.9,
|
|
"mrr": 0.7873,
|
|
"precision": 0.1496,
|
|
"ndcg": 0.814,
|
|
"retrieval_success_rate": 0.9819,
|
|
"avg_schema_tokens": 4621,
|
|
"context_share": 0.1651,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 1.0,
|
|
"customer": 1.0,
|
|
"inventory": 1.0,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9259,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 1.0,
|
|
"hard": 0.9551,
|
|
"medium": 0.9848
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"48": {
|
|
"k": 48,
|
|
"recall": 0.9819,
|
|
"multi_tool_recall": 1.0,
|
|
"mrr": 0.7871,
|
|
"precision": 0.1209,
|
|
"ndcg": 0.818,
|
|
"retrieval_success_rate": 0.9819,
|
|
"avg_schema_tokens": 7862,
|
|
"context_share": 0.2808,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 1.0,
|
|
"customer": 1.0,
|
|
"inventory": 1.0,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9259,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 1.0,
|
|
"hard": 0.9551,
|
|
"medium": 0.9848
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"3": {
|
|
"k": 3,
|
|
"recall": 0.8488,
|
|
"multi_tool_recall": 0.7,
|
|
"mrr": 0.7664,
|
|
"precision": 0.506,
|
|
"ndcg": 0.755,
|
|
"retrieval_success_rate": 0.8465,
|
|
"avg_schema_tokens": 821,
|
|
"context_share": 0.0293,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.6552,
|
|
"accounting": 0.8851,
|
|
"customer": 0.7941,
|
|
"inventory": 0.7833,
|
|
"invoice": 0.86,
|
|
"legacy": 0.9545,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9211,
|
|
"sales": 1.0,
|
|
"support": 0.7037,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 0.8955,
|
|
"hard": 0.809,
|
|
"medium": 0.8223
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 0.6667,
|
|
"conversational": 0.9286,
|
|
"english": 0.9661,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 0.8966
|
|
}
|
|
},
|
|
"8": {
|
|
"k": 8,
|
|
"recall": 0.9345,
|
|
"multi_tool_recall": 0.8,
|
|
"mrr": 0.7828,
|
|
"precision": 0.2499,
|
|
"ndcg": 0.7958,
|
|
"retrieval_success_rate": 0.9323,
|
|
"avg_schema_tokens": 2114,
|
|
"context_share": 0.0755,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.8966,
|
|
"accounting": 0.931,
|
|
"customer": 1.0,
|
|
"inventory": 0.8333,
|
|
"invoice": 1.0,
|
|
"legacy": 0.9545,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.8704,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 0.9627,
|
|
"hard": 0.9213,
|
|
"medium": 0.9188
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 0.9655
|
|
}
|
|
},
|
|
"12": {
|
|
"k": 12,
|
|
"recall": 0.9639,
|
|
"multi_tool_recall": 0.9,
|
|
"mrr": 0.7863,
|
|
"precision": 0.1927,
|
|
"ndcg": 0.8073,
|
|
"retrieval_success_rate": 0.9639,
|
|
"avg_schema_tokens": 3073,
|
|
"context_share": 0.1098,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 0.954,
|
|
"customer": 1.0,
|
|
"inventory": 0.95,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9074,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 0.9851,
|
|
"hard": 0.9551,
|
|
"medium": 0.9543
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"30": {
|
|
"k": 30,
|
|
"recall": 0.9819,
|
|
"multi_tool_recall": 0.9,
|
|
"mrr": 0.7872,
|
|
"precision": 0.1308,
|
|
"ndcg": 0.815,
|
|
"retrieval_success_rate": 0.9819,
|
|
"avg_schema_tokens": 6072,
|
|
"context_share": 0.2169,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 1.0,
|
|
"customer": 1.0,
|
|
"inventory": 1.0,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9259,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 1.0,
|
|
"hard": 0.9551,
|
|
"medium": 0.9848
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"64": {
|
|
"k": 64,
|
|
"recall": 0.9819,
|
|
"multi_tool_recall": 1.0,
|
|
"mrr": 0.7871,
|
|
"precision": 0.1195,
|
|
"ndcg": 0.8185,
|
|
"retrieval_success_rate": 0.9819,
|
|
"avg_schema_tokens": 8522,
|
|
"context_share": 0.3043,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 1.0,
|
|
"customer": 1.0,
|
|
"inventory": 1.0,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9259,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 1.0,
|
|
"hard": 0.9551,
|
|
"medium": 0.9848
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
},
|
|
"128": {
|
|
"k": 128,
|
|
"recall": 0.9819,
|
|
"multi_tool_recall": 1.0,
|
|
"mrr": 0.7871,
|
|
"precision": 0.1194,
|
|
"ndcg": 0.8185,
|
|
"retrieval_success_rate": 0.9819,
|
|
"avg_schema_tokens": 8582,
|
|
"context_share": 0.3065,
|
|
"no_match_success": 0.6333,
|
|
"no_match_false_positive": 0.3667,
|
|
"no_match_false_negative": 0.0,
|
|
"per_category_recall": {
|
|
"CRM": 0.931,
|
|
"accounting": 1.0,
|
|
"customer": 1.0,
|
|
"inventory": 1.0,
|
|
"invoice": 1.0,
|
|
"legacy": 1.0,
|
|
"memory": 1.0,
|
|
"purchase": 1.0,
|
|
"reports": 0.9474,
|
|
"sales": 1.0,
|
|
"support": 0.9259,
|
|
"wallet": 1.0,
|
|
"workflow": 1.0
|
|
},
|
|
"per_difficulty_recall": {
|
|
"ambiguous": 0.9565,
|
|
"easy": 1.0,
|
|
"hard": 0.9551,
|
|
"medium": 0.9848
|
|
},
|
|
"per_hard_case_recall": {
|
|
"ambiguous": 1.0,
|
|
"conversational": 1.0,
|
|
"english": 0.9831,
|
|
"generic": 0.6667,
|
|
"long": 1.0,
|
|
"multi-intent": 1.0,
|
|
"persian_accounting": 1.0,
|
|
"short": 1.0,
|
|
"similar_tools": 1.0,
|
|
"synonyms": 1.0,
|
|
"typos": 1.0
|
|
}
|
|
}
|
|
},
|
|
"recommended": {
|
|
"decision": "A",
|
|
"recommended_analyzer_k": 10,
|
|
"recall_at_10": 0.9549,
|
|
"recall_at_15": 0.9752,
|
|
"recall_at_20": 0.9819,
|
|
"recall_at_48": 0.9819,
|
|
"note": "K=10 holds high recall; Tight Top-K is justified.",
|
|
"keep_autonomous_k": 128
|
|
},
|
|
"adaptive": {
|
|
"queries": 443,
|
|
"retrieval_success": 0.9481,
|
|
"avg_k": 8.81,
|
|
"k_histogram": {
|
|
"5": 297,
|
|
"10": 50,
|
|
"20": 96
|
|
}
|
|
},
|
|
"iteration": {
|
|
"schema_resent_every_iteration": true,
|
|
"complexity_iterations": {
|
|
"simple": 3,
|
|
"medium": 6,
|
|
"complex": 12
|
|
},
|
|
"avg_iterations_at_k15": 4.98,
|
|
"avg_schema_tokens_per_iteration_k15": 3706,
|
|
"avg_total_schema_tokens_per_request_k15": 19931,
|
|
"avg_total_schema_tokens_per_request_k48": 42977
|
|
},
|
|
"ci_gates": {
|
|
"recall_at_10_min": 0.9,
|
|
"recall_at_20_min": 0.93,
|
|
"recall_at_48_min": 0.93,
|
|
"legacy_recall_at_48_min": 0.95,
|
|
"retrieval_success_at_15_min": 0.92
|
|
},
|
|
"failure_count_at_15": 11,
|
|
"failures_at_15": [
|
|
{
|
|
"id": "exp-044",
|
|
"query": "امتیاز باشگاه را کم و زیاد کن",
|
|
"expected": [
|
|
"adjust_customer_club_points"
|
|
],
|
|
"returned_top": [
|
|
"get_customer_club_settings",
|
|
"list_customer_club_ledger",
|
|
"get_customer_club_rfm_summary",
|
|
"list_customer_club_tiers",
|
|
"search_customer_club_rfm_persons",
|
|
"get_inventory_status",
|
|
"search_invoices",
|
|
"search_persons",
|
|
"get_document_details",
|
|
"get_sales_report",
|
|
"get_workflow",
|
|
"list_workflow_trigger_catalog"
|
|
],
|
|
"wrong_top": "get_customer_club_settings",
|
|
"reason": "missing_primary:adjust_customer_club_points;wrong_top:get_customer_club_settings",
|
|
"category": "CRM",
|
|
"difficulty": "hard",
|
|
"hard_case": "",
|
|
"top_score": 29,
|
|
"score_gap": 11
|
|
},
|
|
{
|
|
"id": "exp-046",
|
|
"query": "RFM باشگاه را دوباره حساب کن",
|
|
"expected": [
|
|
"recalculate_customer_club_rfm"
|
|
],
|
|
"returned_top": [
|
|
"get_customer_club_rfm_summary",
|
|
"get_customer_club_settings",
|
|
"list_customer_club_ledger",
|
|
"get_account",
|
|
"list_bank_accounts",
|
|
"search_customer_club_rfm_persons",
|
|
"search_invoices",
|
|
"get_document_details",
|
|
"get_sales_report",
|
|
"list_accounts",
|
|
"list_customer_club_tiers",
|
|
"search_persons"
|
|
],
|
|
"wrong_top": "get_customer_club_rfm_summary",
|
|
"reason": "missing_primary:recalculate_customer_club_rfm",
|
|
"category": "CRM",
|
|
"difficulty": "hard",
|
|
"hard_case": "",
|
|
"top_score": 28,
|
|
"score_gap": 11
|
|
},
|
|
{
|
|
"id": "exp-058",
|
|
"query": "قالب گزارش را منتشر کن",
|
|
"expected": [
|
|
"publish_report_template"
|
|
],
|
|
"returned_top": [
|
|
"list_report_templates",
|
|
"get_report",
|
|
"get_report_template",
|
|
"get_report_template_scope_catalog",
|
|
"get_sales_report",
|
|
"search_invoices",
|
|
"search_persons",
|
|
"get_document_details",
|
|
"get_pipeline_report",
|
|
"get_purchase_report",
|
|
"get_workflow",
|
|
"list_available_reports"
|
|
],
|
|
"wrong_top": "list_report_templates",
|
|
"reason": "missing_primary:publish_report_template",
|
|
"category": "reports",
|
|
"difficulty": "hard",
|
|
"hard_case": "",
|
|
"top_score": 24,
|
|
"score_gap": 11
|
|
},
|
|
{
|
|
"id": "exp-086",
|
|
"query": "connector call",
|
|
"expected": [
|
|
"invoke_business_connector"
|
|
],
|
|
"returned_top": [
|
|
"get_basalam_overview",
|
|
"list_basalam_dead_letter",
|
|
"list_basalam_product_conflicts",
|
|
"list_basalam_synced_invoices",
|
|
"list_woocommerce_orders",
|
|
"list_woocommerce_products"
|
|
],
|
|
"wrong_top": "get_basalam_overview",
|
|
"reason": "missing_primary:invoke_business_connector;wrong_top:get_basalam_overview",
|
|
"category": "support",
|
|
"difficulty": "hard",
|
|
"hard_case": "english",
|
|
"top_score": 5,
|
|
"score_gap": 0
|
|
},
|
|
{
|
|
"id": "acc-18",
|
|
"query": "این چک را حذف کن",
|
|
"expected": [
|
|
"delete_check"
|
|
],
|
|
"returned_top": [
|
|
"search_checks",
|
|
"delete_invoice",
|
|
"delete_receipt_payment",
|
|
"get_sales_report",
|
|
"search_invoices",
|
|
"get_check_details",
|
|
"get_document_details",
|
|
"get_invoice_details",
|
|
"delete_memory_entry",
|
|
"delete_person",
|
|
"delete_workflow",
|
|
"get_purchase_report"
|
|
],
|
|
"wrong_top": "search_checks",
|
|
"reason": "missing_primary:delete_check",
|
|
"category": "accounting",
|
|
"difficulty": "easy",
|
|
"hard_case": "",
|
|
"top_score": 15,
|
|
"score_gap": 3
|
|
},
|
|
{
|
|
"id": "con-01",
|
|
"query": "کانکتور قیمت را صدا بزن",
|
|
"expected": [
|
|
"invoke_business_connector"
|
|
],
|
|
"returned_top": [
|
|
"get_basalam_overview",
|
|
"list_basalam_dead_letter",
|
|
"list_basalam_product_conflicts",
|
|
"list_basalam_synced_invoices",
|
|
"list_woocommerce_orders",
|
|
"list_woocommerce_products",
|
|
"list_price_list_items",
|
|
"create_invoice",
|
|
"list_price_lists",
|
|
"search_invoices",
|
|
"search_persons",
|
|
"adjust_customer_club_points"
|
|
],
|
|
"wrong_top": "get_basalam_overview",
|
|
"reason": "missing_primary:invoke_business_connector;wrong_top:get_basalam_overview",
|
|
"category": "support",
|
|
"difficulty": "medium",
|
|
"hard_case": "",
|
|
"top_score": 5,
|
|
"score_gap": 0
|
|
},
|
|
{
|
|
"id": "exp-028",
|
|
"query": "این سرفصل را حذف کن",
|
|
"expected": [
|
|
"delete_account"
|
|
],
|
|
"returned_top": [
|
|
"list_accounts",
|
|
"get_account",
|
|
"delete_invoice",
|
|
"delete_receipt_payment",
|
|
"get_sales_report",
|
|
"search_invoices",
|
|
"get_document_details",
|
|
"get_invoice_details",
|
|
"delete_memory_entry",
|
|
"delete_person",
|
|
"delete_workflow",
|
|
"get_purchase_report"
|
|
],
|
|
"wrong_top": "list_accounts",
|
|
"reason": "missing_primary:delete_account",
|
|
"category": "accounting",
|
|
"difficulty": "medium",
|
|
"hard_case": "",
|
|
"top_score": 17,
|
|
"score_gap": 1
|
|
},
|
|
{
|
|
"id": "exp-031",
|
|
"query": "انتقال بانکی را حذف کن",
|
|
"expected": [
|
|
"delete_transfer"
|
|
],
|
|
"returned_top": [
|
|
"delete_invoice",
|
|
"search_transfers",
|
|
"delete_receipt_payment",
|
|
"list_bank_accounts",
|
|
"search_invoices",
|
|
"get_document_details",
|
|
"get_sales_report",
|
|
"get_invoice_details",
|
|
"resolve_currency_rate",
|
|
"delete_account",
|
|
"delete_check",
|
|
"delete_expense_income"
|
|
],
|
|
"wrong_top": "delete_invoice",
|
|
"reason": "missing_primary:delete_transfer;wrong_top:delete_invoice",
|
|
"category": "accounting",
|
|
"difficulty": "medium",
|
|
"hard_case": "",
|
|
"top_score": 11,
|
|
"score_gap": 2
|
|
},
|
|
{
|
|
"id": "exp-158",
|
|
"query": "اطلاعات کسبوکار نشست",
|
|
"expected": [
|
|
"get_business_info"
|
|
],
|
|
"returned_top": [
|
|
"list_business_users",
|
|
"get_business_credit_settings",
|
|
"get_business_dashboard",
|
|
"get_wallet_overview",
|
|
"list_my_businesses"
|
|
],
|
|
"wrong_top": "list_business_users",
|
|
"reason": "missing_primary:get_business_info;wrong_top:list_business_users",
|
|
"category": "support",
|
|
"difficulty": "medium",
|
|
"hard_case": "",
|
|
"top_score": 4,
|
|
"score_gap": 2
|
|
},
|
|
{
|
|
"id": "hard-gen-01",
|
|
"query": "یک گزارش کلی از اوضاع بده",
|
|
"expected": [
|
|
"get_business_dashboard"
|
|
],
|
|
"returned_top": [
|
|
"get_report",
|
|
"get_sales_report",
|
|
"get_creditors_report",
|
|
"get_pipeline_report",
|
|
"get_purchase_report",
|
|
"list_available_reports",
|
|
"get_debtors_report",
|
|
"get_inventory_status",
|
|
"get_invoice_details",
|
|
"get_report_template",
|
|
"list_report_templates",
|
|
"list_workflow_executions"
|
|
],
|
|
"wrong_top": "get_report",
|
|
"reason": "missing_primary:get_business_dashboard;alias_gap:get_business_dashboard",
|
|
"category": "reports",
|
|
"difficulty": "ambiguous",
|
|
"hard_case": "generic",
|
|
"top_score": 13,
|
|
"score_gap": 9
|
|
},
|
|
{
|
|
"id": "plat-01",
|
|
"query": "اطلاعات این کسبوکار",
|
|
"expected": [
|
|
"get_business_info"
|
|
],
|
|
"returned_top": [
|
|
"list_business_users",
|
|
"get_business_credit_settings",
|
|
"get_business_dashboard",
|
|
"get_sales_report",
|
|
"get_wallet_overview",
|
|
"list_my_businesses",
|
|
"get_invoice_details",
|
|
"get_purchase_report",
|
|
"search_expense_income",
|
|
"search_invoices",
|
|
"search_leads",
|
|
"search_transfers"
|
|
],
|
|
"wrong_top": "list_business_users",
|
|
"reason": "missing_primary:get_business_info;wrong_top:list_business_users",
|
|
"category": "support",
|
|
"difficulty": "medium",
|
|
"hard_case": "",
|
|
"top_score": 5,
|
|
"score_gap": 3
|
|
}
|
|
]
|
|
} |