Watch
1
0
Fork
You've already forked arc
0
forked from hesabix/arc
arc/hesabixAPI/app/services/ai/ai_tool_scale_bench.py
2026-08-21 00:15:57 +00:00

257 lines
8.7 KiB
Python

"""Synthetic catalog scale benchmark — never registers into production."""
from __future__ import annotations
import json
import time
import tracemalloc
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
from typing import Any, Dict, List
from app.services.ai.ai_embedding_service import cosine_similarity
from app.services.ai.ai_tool_candidates import retrieve_candidates
from app.services.ai.ai_tool_embedding import hashed_ngram_vector
from app.services.ai.ai_tool_manifest import get_manifest_entry, iter_manifest_names
from app.services.ai.ai_tool_rank import score_tool_for_query
from app.services.ai.ai_tool_retrieval_index import ToolRetrievalIndex
from app.services.ai.ai_tool_schema import (
SchemaSource,
load_tool_schemas,
reset_schema_cache_for_tests,
)
BENCH_DIR = Path("/opt/hesabix/app/docs/ai-agent/evals/tool-scale")
SIZES = (180, 500, 1200, 5000, 10000)
LARGE_SIZE = 100000
QUERY = "فاکتور فروش این ماه را پیدا کن"
STRESS_QPS = (1, 10, 50, 100)
class _FakeSchema(SchemaSource):
def schema_version_for(self, name: str) -> str:
return "1"
def build_openai_tool_definition(self, name: str):
return {
"type": "function",
"function": {
"name": name,
"description": f"synthetic or real tool {name}",
"parameters": {"type": "object", "properties": {}},
},
}
def _catalog(size: int) -> List[str]:
real = list(iter_manifest_names())
if size <= len(real):
return real[:size]
extra = [f"synth_tool_{i:05d}" for i in range(size - len(real))]
return real + extra
def _search_text(name: str) -> str:
entry = get_manifest_entry(name)
if entry is not None:
return entry.search_text(name, "")
try:
n = int(name.rsplit("_", 1)[-1])
except ValueError:
n = 1
if n % 47 == 0:
return f"{name} invoice فاکتور فروش"
return f"{name} synthetic warehouse crm report search list"
def _build_index(names: List[str]) -> ToolRetrievalIndex:
index = ToolRetrievalIndex()
docs = []
for name in names:
extra = "" if get_manifest_entry(name) is not None else _search_text(name)
docs.append((name, extra))
index.load_many(docs)
return index
def _percentile(values: List[float], p: float) -> float:
if not values:
return 0.0
ordered = sorted(values)
idx = int(round((p / 100.0) * (len(ordered) - 1)))
idx = max(0, min(len(ordered) - 1, idx))
return ordered[idx]
def _bench_size(
size: int,
*,
include_semantic: bool = True,
include_linear: bool = True,
) -> Dict[str, Any]:
names = _catalog(size)
index = _build_index(names)
stats = index.stats()
lex_ms = None
ranked: List[str] = []
if include_linear:
t0 = time.perf_counter()
lex = {n: score_tool_for_query(n, QUERY) for n in names}
lex_ms = round((time.perf_counter() - t0) * 1000.0, 3)
ranked = sorted(lex, key=lambda n: (-lex[n], n))[:15]
query_latencies = []
for _ in range(12):
offer = retrieve_candidates(
QUERY,
names,
n=100,
fallback_on_empty=False,
include_vector=False,
index=index,
expects_tools=True,
)
query_latencies.append(offer.latency_ms)
inv_ms = round(sum(query_latencies) / len(query_latencies), 3)
sem_ms = None
if include_semantic:
vectors = {n: hashed_ngram_vector(_search_text(n)) for n in names}
qvec = hashed_ngram_vector(QUERY)
t1 = time.perf_counter()
sem = []
for n, vec in vectors.items():
s = cosine_similarity(qvec, vec)
if s > 0.02:
sem.append((n, s))
sem.sort(key=lambda item: -item[1])
sem_ms = round((time.perf_counter() - t1) * 1000.0, 3)
schema_ms = None
if ranked:
reset_schema_cache_for_tests()
t2 = time.perf_counter()
load_tool_schemas(
ranked,
authorized_names=set(ranked),
source=_FakeSchema(),
)
schema_ms = round((time.perf_counter() - t2) * 1000.0, 3)
return {
"size": size,
"real_tools": min(size, len(iter_manifest_names())),
"synthetic_tools": max(0, size - len(iter_manifest_names())),
"keyword_linear_ms": lex_ms,
"inverted_index_build_ms": stats.build_ms,
"inverted_index_query_ms": inv_ms,
"inverted_index_query_p50_ms": round(_percentile(query_latencies, 50), 3),
"inverted_index_query_p95_ms": round(_percentile(query_latencies, 95), 3),
"inverted_postings": stats.postings,
"index_memory_bytes_est": stats.memory_bytes_est,
"semantic_linear_ms": sem_ms,
"ranking_latency_ms": lex_ms,
"semantic_search_latency_ms": sem_ms,
"discovery_latency_ms": round((lex_ms or 0) + (sem_ms or 0), 3),
"schema_loading_latency_ms": schema_ms,
"database_queries": 0,
"top15": ranked[:5],
"inverted_top15": list(offer.names[:5]),
"production_ready": False,
}
def _stress(index: ToolRetrievalIndex, names: List[str], workers: int) -> Dict[str, Any]:
latencies: List[float] = []
def _one() -> float:
offer = retrieve_candidates(
QUERY,
names,
n=100,
fallback_on_empty=False,
include_vector=False,
index=index,
expects_tools=True,
)
return offer.latency_ms
started = time.perf_counter()
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
futs = [pool.submit(_one) for _ in range(max(workers, 1))]
for fut in as_completed(futs):
latencies.append(fut.result())
wall = round((time.perf_counter() - started) * 1000.0, 3)
return {
"concurrency": workers,
"queries": len(latencies),
"wall_ms": wall,
"p50_ms": round(_percentile(latencies, 50), 3),
"p95_ms": round(_percentile(latencies, 95), 3),
}
def run_scale_benchmark(*, write_files: bool = False) -> Dict[str, Any]:
tracemalloc.start()
rows = [_bench_size(size) for size in SIZES]
large = _bench_size(LARGE_SIZE, include_semantic=False, include_linear=False)
names_10k = _catalog(10000)
index_10k = _build_index(names_10k)
stress = [_stress(index_10k, names_10k, qps) for qps in STRESS_QPS]
current, peak = tracemalloc.get_traced_memory()
tracemalloc.stop()
ten_k = next((row for row in rows if row["size"] == 10000), {})
report = {
"sizes": rows,
"size_100000": large,
"stress_10k": stress,
"memory_current_kb": round(current / 1024, 1),
"memory_peak_kb": round(peak / 1024, 1),
"note": (
"Synthetic names never enter production registry. "
"Inverted index is derived posting lists. "
"Linear semantic at 10k is not a production design. "
"100k skips semantic scan."
),
"index_study": {
"A_inverted_keyword": "In-process inverted index is the Phase 10 candidate generator.",
"B_postgres_gin_trigram": "Optional later for fuzzy tokens; not required at 10k.",
"C_pgvector_ann": "Existing pgvector is for knowledge chunks, not tools. No new vector DB.",
"D_hnsw": "Preferred future ANN when a versioned neural embedding exists.",
"E_hybrid_inverted_plus_vector": "Merge is implemented behind VECTOR_CANDIDATE_RETRIEVAL=False.",
"phase_10_gate": "Do not ship 10k linear semantic. Keyword inverted index is the online path.",
"semantic_10k_ms": ten_k.get("semantic_linear_ms"),
"inverted_10k_query_ms": ten_k.get("inverted_index_query_ms"),
"keyword_10k_ms": ten_k.get("keyword_linear_ms"),
},
}
if write_files:
BENCH_DIR.mkdir(parents=True, exist_ok=True)
(BENCH_DIR / "latest.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
return report
def main() -> None:
report = run_scale_benchmark(write_files=True)
summary = {
"sizes": [
{
"size": row["size"],
"inverted_ms": row["inverted_index_query_ms"],
"keyword_ms": row["keyword_linear_ms"],
"semantic_ms": row["semantic_linear_ms"],
"build_ms": row["inverted_index_build_ms"],
}
for row in report["sizes"]
],
"size_100000_inverted_ms": report["size_100000"]["inverted_index_query_ms"],
"stress_10k": report["stress_10k"],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()