Seyyed/Seyyed_arc
Watch
1
0
Fork
You've already forked Seyyed_arc
 0
forked from hesabix/arc
Code Pull requests Activity
Seyyed_arc/docs/ai-agent/evals/phase-14/latest.json
Babak Alizadeh a1a3df72e8 progress in ai
2026-08-21 00:15:57 +00:00

582 lines
No EOL
13 KiB
JSON
Raw Permalink Blame History

{
"offline_gold": {
"source": "offline_gold",
"not_production": true,
"candidate_recall_at_100": 1.0,
"indexed_final": [
{
"k": 10,
"candidate_n": 100,
"recall": 0.9616,
"mrr": 0.7931,
"ndcg": 0.8091
},
{
"k": 15,
"candidate_n": 100,
"recall": 0.9819,
"mrr": 0.7945,
"ndcg": 0.8166
},
{
"k": 20,
"candidate_n": 100,
"recall": 0.9887,
"mrr": 0.7948,
"ndcg": 0.8191
},
{
"k": 48,
"candidate_n": 100,
"recall": 0.9887,
"mrr": 0.7947,
"ndcg": 0.8233
}
],
"baseline_final": [
{
"k": 10,
"recall": 0.9616
},
{
"k": 15,
"recall": 0.9819
},
{
"k": 20,
"recall": 0.9887
},
{
"k": 48,
"recall": 0.9887
}
],
"note": "Copied from Phase 13 offline Gold. Not Production Recall. Not Production Control or Experiment."
},
"offline_performance": {
"source": "offline_performance_evidence",
"not_production": true,
"indexed_10k_ms": 6,
"full_rank_10k_ms": 110,
"note": "Synthetic catalog bench from Phase 10/11. Measure production latency separately."
},
"production_control": {
"source": "production_control",
"requests": 0
},
"production_experiment": {
"source": "production_experiment",
"requests": 0
},
"production_live": {
"source": "production_live",
"offline_gold_mixed": false,
"production_control": {
"source": "production_control",
"requests": 0
},
"production_experiment": {
"source": "production_experiment",
"requests": 0
},
"control_requests": 0,
"experiment_requests": 0,
"live_control_requests": 0,
"live_experiment_requests": 0,
"experiment_enabled_requests": 0,
"experiment_fallback_requests": 0,
"exposure": 0,
"indexed_retrieval_hit": 0,
"indexed_retrieval_miss": 0,
"indexed_retrieval_empty": 0,
"indexed_retrieval_stale": 0,
"indexed_retrieval_error": 0,
"indexed_hit_rate": null,
"indexed_miss_rate": null,
"fallback_rate": null,
"fallback_total": 0,
"fallback_empty": 0,
"fallback_low_candidate_quality": 0,
"fallback_stale_index": 0,
"fallback_index_error": 0,
"fallback_index_inconsistency": 0,
"fallback_reasons": {},
"task_success": null,
"task_success_source": null,
"first_pass_without_rediscovery": null,
"index_success": null,
"tool_execution_success": null,
"tool_execution": {
"read": {
"control": {
"success": 0,
"failure": 0
},
"experiment": {
"success": 0,
"failure": 0
}
},
"write": {
"control": {
"success": 0,
"failure": 0
},
"experiment": {
"success": 0,
"failure": 0
}
},
"delete": {
"control": {
"success": 0,
"failure": 0
},
"experiment": {
"success": 0,
"failure": 0
}
},
"export": {
"control": {
"success": 0,
"failure": 0
},
"experiment": {
"success": 0,
"failure": 0
}
},
"execute": {
"control": {
"success": 0,
"failure": 0
},
"experiment": {
"success": 0,
"failure": 0
}
}
},
"no_match_requests": 0,
"no_match_false_tool_selection": 0,
"security": {
"authorization_bypass": 0,
"tenant_leakage": 0,
"permission_violation": 0
},
"latency": {
"retrieval": {
"control": {
"p50": 0,
"p95": 0,
"p99": 0
},
"experiment": {
"p50": 0,
"p95": 0,
"p99": 0
}
},
"end_to_end": {
"control": {
"p50": 0,
"p95": 0,
"p99": 0
},
"experiment": {
"p50": 0,
"p95": 0,
"p99": 0
}
},
"discovery": {
"control": {
"p50": 0,
"p95": 0,
"p99": 0,
"avg_ms": 0.0
},
"experiment": {
"p50": 0,
"p95": 0,
"p99": 0,
"avg_ms": 0.0
}
},
"index": {
"control": {
"p50": 0,
"p95": 0,
"p99": 0
},
"experiment": {
"p50": 0,
"p95": 0,
"p99": 0
}
},
"rank": {
"control": {
"p50": 0,
"p95": 0,
"p99": 0
},
"experiment": {
"p50": 0,
"p95": 0,
"p99": 0
}
},
"total_ai_request": {
"control": {
"p50": 0,
"p95": 0,
"p99": 0
},
"experiment": {
"p50": 0,
"p95": 0,
"p99": 0
}
},
"p50": null,
"p95": null,
"p99": null
},
"by_channel": {
"control": {},
"experiment": {},
"channels": [
"chat",
"subagent",
"crm",
"ticket",
"workflow"
]
},
"high_risk": {
"delete": {
"control": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
},
"experiment": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
}
},
"write": {
"control": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
},
"experiment": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
}
},
"export": {
"control": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
},
"experiment": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
}
},
"execute": {
"control": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
},
"experiment": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
}
},
"financial_mutation": {
"control": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
},
"experiment": {
"success": 0,
"failure": 0,
"approval": 0,
"approval_success": 0,
"unexpected_rejection": 0,
"authorization_failure": 0
}
}
},
"index_health": {
"active_tool_count": 180,
"indexed_tool_count": 180,
"stale_tool_count": 0,
"disabled_tool_count": 0,
"last_refresh": 1787270721.1194706,
"healthy": true,
"active_tools": 180,
"indexed_tools": 180,
"stale_tools": 0,
"disabled_tools": 0,
"index_refresh_age": 0.004,
"index_health": "healthy",
"index_consistency_failures": 0
},
"comparison": {
"task_completion": {
"control": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"experiment": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"delta": null,
"ci_overlap": null,
"enough_for_significance": false
},
"first_pass_without_rediscovery": {
"control": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"experiment": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"delta": null,
"ci_overlap": null,
"enough_for_significance": false
},
"fallback_rate": {
"control": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"experiment": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"delta": null,
"ci_overlap": null,
"enough_for_significance": false
},
"indexed_retrieval_miss": {
"control": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"experiment": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"delta": null,
"ci_overlap": null,
"enough_for_significance": false
},
"tool_success": {
"control": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"experiment": {
"n": 0,
"rate": null,
"ci95": [
null,
null
]
},
"delta": null,
"ci_overlap": null,
"enough_for_significance": false
}
},
"promotion": {
"control_requests": 0,
"experiment_requests": 0,
"min_samples": 500,
"auto_promotion": false,
"allow_operator_promote": false,
"security_events": 0,
"gates": {
"A_security": "pass",
"B_task": "unknown",
"C_index_miss": "unknown",
"D_fallback": "unknown",
"E_latency": "unknown"
},
"decision": "HOLD",
"case": "insufficient_samples",
"reason": "need_500_per_arm"
},
"rollout": {
"current_percent": 5,
"ladder": [
5,
10,
25,
50,
100
],
"next_percent": 10,
"auto_promotion": false,
"promotion_requires": "explicit_operator_decision",
"global_flag": false,
"killed": false
},
"flags": {
"INDEXED_CANDIDATE_RETRIEVAL": false,
"INDEXED_CANDIDATE_ROLLOUT_PERCENT": 5,
"INDEXED_CANDIDATE_FALLBACK": true,
"INDEXED_CANDIDATE_KILL_SWITCH": false,
"ADAPTIVE_DISCOVERY_K": false,
"HYBRID_TOOL_DISCOVERY": false,
"PROGRESSIVE_SCHEMA_LOADING": false,
"min_live_samples": 500,
"rollout_ladder": [
5,
10,
25,
50,
100
]
},
"note": "Production Live only. Offline gold metrics are not included. Empty rates mean no live traffic — HOLD, do not fabricate. task_success is never inferred from tool_execution_success."
},
"flags": {
"INDEXED_CANDIDATE_RETRIEVAL": false,
"INDEXED_CANDIDATE_ROLLOUT_PERCENT": 5,
"INDEXED_CANDIDATE_FALLBACK": true,
"INDEXED_CANDIDATE_KILL_SWITCH": false,
"ADAPTIVE_DISCOVERY_K": false,
"HYBRID_TOOL_DISCOVERY": false,
"PROGRESSIVE_SCHEMA_LOADING": false,
"CANDIDATE_RETRIEVAL_N": 100,
"MAX_TOOLS_PER_REQUEST": 48,
"MAX_TOOLS_AUTONOMOUS": 128,
"min_live_samples": 500,
"rollout_ladder": [
5,
10,
25,
50,
100
]
},
"index_health": {
"active_tool_count": 180,
"indexed_tool_count": 180,
"stale_tool_count": 0,
"disabled_tool_count": 0,
"last_refresh": 1787270721.1194706,
"healthy": true,
"active_tools": 180,
"indexed_tools": 180,
"stale_tools": 0,
"disabled_tools": 0,
"index_refresh_age": 0.008,
"index_health": "healthy"
},
"promotion": {
"control_requests": 0,
"experiment_requests": 0,
"min_samples": 500,
"auto_promotion": false,
"allow_operator_promote": false,
"security_events": 0,
"gates": {
"A_security": "pass",
"B_task": "unknown",
"C_index_miss": "unknown",
"D_fallback": "unknown",
"E_latency": "unknown"
},
"decision": "HOLD",
"case": "insufficient_samples",
"reason": "need_500_per_arm"
},
"decision": {
"promotion": "HOLD",
"reason": "need_500_per_arm",
"auto_promotion": false
},
"note": "No live evidence, no promotion. Do not interpret fallback-assisted success as index success. Do not mix Indexed Retrieval validation with Adaptive K validation."
}
View git blame Copy permalink