192 lines
6.9 KiB
Python
192 lines
6.9 KiB
Python
|
import sys
|
||
|
|
import os
|
||
|
|
import json
|
||
|
|
import requests
|
||
|
|
import time
|
||
|
|
import numpy as np
|
||
|
|
from time import sleep
|
||
|
|
|
||
|
|
# 将项目根目录加入路径
|
||
|
|
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
|
from backend.core.config import settings
|
||
|
|
|
||
|
|
# ================= ⚙️ 配置区域 =================
|
||
|
|
BASE_URL = "http://127.0.0.1:8000"
|
||
|
|
TASK_ID = 19 # ⚠️ 请修改为你实际爬取数据的 Task ID
|
||
|
|
# 自动适配操作系统路径
|
||
|
|
TEST_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_dataset.json")
|
||
|
|
# ==============================================
|
||
|
|
|
||
|
|
class Colors:
|
||
|
|
HEADER = '\033[95m'
|
||
|
|
OKBLUE = '\033[94m'
|
||
|
|
OKCYAN = '\033[96m'
|
||
|
|
OKGREEN = '\033[92m'
|
||
|
|
WARNING = '\033[93m'
|
||
|
|
FAIL = '\033[91m'
|
||
|
|
ENDC = '\033[0m'
|
||
|
|
BOLD = '\033[1m'
|
||
|
|
|
||
|
|
def get_rag_results(query):
|
||
|
|
"""
|
||
|
|
调用搜索接口并记录耗时
|
||
|
|
"""
|
||
|
|
start_ts = time.time()
|
||
|
|
try:
|
||
|
|
# 调用 V2 接口，该接口内部已集成 混合检索 -> Rerank
|
||
|
|
res = requests.post(
|
||
|
|
f"{BASE_URL}/api/v2/search",
|
||
|
|
json={"query": query, "task_id": TASK_ID, "limit": 5}, # 获取 Top 5
|
||
|
|
timeout=15
|
||
|
|
)
|
||
|
|
latency = (time.time() - start_ts) * 1000 # ms
|
||
|
|
|
||
|
|
if res.status_code != 200:
|
||
|
|
print(f"{Colors.FAIL}❌ API Error {res.status_code}: {res.text}{Colors.ENDC}")
|
||
|
|
return [], 0
|
||
|
|
|
||
|
|
res_json = res.json()
|
||
|
|
chunks = res_json.get('data', {}).get('results', [])
|
||
|
|
return chunks, latency
|
||
|
|
except Exception as e:
|
||
|
|
print(f"{Colors.FAIL}❌ 请求异常: {e}{Colors.ENDC}")
|
||
|
|
return [], 0
|
||
|
|
|
||
|
|
def check_hit(content, keywords):
|
||
|
|
"""
|
||
|
|
检查切片相关性 (Relevance Check)
|
||
|
|
使用关键词匹配作为 Ground Truth 的轻量级验证。
|
||
|
|
"""
|
||
|
|
if not keywords: return True # 拒答题或开放性题目跳过关键词检查
|
||
|
|
if not content: return False
|
||
|
|
|
||
|
|
content_lower = content.lower()
|
||
|
|
for k in keywords:
|
||
|
|
if k.lower() in content_lower:
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def run_evaluation():
|
||
|
|
# 1. 加载测试集
|
||
|
|
if not os.path.exists(TEST_FILE):
|
||
|
|
print(f"{Colors.FAIL}❌ 找不到测试文件: {TEST_FILE}{Colors.ENDC}")
|
||
|
|
print("请确保 scripts/test_dataset.json 文件存在。")
|
||
|
|
return
|
||
|
|
|
||
|
|
with open(TEST_FILE, 'r', encoding='utf-8') as f:
|
||
|
|
dataset = json.load(f)
|
||
|
|
|
||
|
|
print(f"{Colors.HEADER}🚀 开始全维度量化评测 (Task ID: {TASK_ID}){Colors.ENDC}")
|
||
|
|
print(f"📄 测试集包含 {len(dataset)} 个样本\n")
|
||
|
|
|
||
|
|
# === 统计容器 ===
|
||
|
|
metrics = {
|
||
|
|
"p_at_1": [], # Precision@1: 正确答案排第1
|
||
|
|
"hit_at_5": [], # HitRate@5: 正确答案在前5
|
||
|
|
"mrr": [], # MRR: 倒数排名分数
|
||
|
|
"latency": [] # 耗时
|
||
|
|
}
|
||
|
|
|
||
|
|
# === 开始循环测试 ===
|
||
|
|
for i, item in enumerate(dataset):
|
||
|
|
query = item['query']
|
||
|
|
print(f"📝 Case {i+1}: {Colors.BOLD}{query}{Colors.ENDC}")
|
||
|
|
|
||
|
|
# 执行检索
|
||
|
|
chunks, latency = get_rag_results(query)
|
||
|
|
metrics['latency'].append(latency)
|
||
|
|
|
||
|
|
# 计算单次指标
|
||
|
|
is_hit_at_5 = 0
|
||
|
|
p_at_1 = 0
|
||
|
|
reciprocal_rank = 0.0
|
||
|
|
hit_position = -1
|
||
|
|
hit_chunk = None
|
||
|
|
|
||
|
|
# 遍历 Top 5 结果
|
||
|
|
for idx, chunk in enumerate(chunks):
|
||
|
|
if check_hit(chunk['content'], item['keywords']):
|
||
|
|
# 命中！
|
||
|
|
is_hit_at_5 = 1
|
||
|
|
hit_position = idx
|
||
|
|
reciprocal_rank = 1.0 / (idx + 1)
|
||
|
|
hit_chunk = chunk
|
||
|
|
|
||
|
|
# 如果是第1个就命中了
|
||
|
|
if idx == 0:
|
||
|
|
p_at_1 = 1
|
||
|
|
|
||
|
|
# 找到即停止 (MRR计算只需知道第一个正确答案的位置)
|
||
|
|
break
|
||
|
|
|
||
|
|
# 记录指标
|
||
|
|
metrics['p_at_1'].append(p_at_1)
|
||
|
|
metrics['hit_at_5'].append(is_hit_at_5)
|
||
|
|
metrics['mrr'].append(reciprocal_rank)
|
||
|
|
|
||
|
|
# 打印单行结果
|
||
|
|
if is_hit_at_5:
|
||
|
|
rank_display = f"Rank {hit_position + 1}"
|
||
|
|
color = Colors.OKGREEN if hit_position == 0 else Colors.OKCYAN
|
||
|
|
source = hit_chunk.get('source_url', 'Unknown')
|
||
|
|
|
||
|
|
# 跨语言污染检查 (简单规则)
|
||
|
|
warning = ""
|
||
|
|
if "/es/" in source and "Spanish" not in query: warning = f"{Colors.WARNING}[跨语言风险]{Colors.ENDC}"
|
||
|
|
elif "/zh/" in source and "如何" not in query and "什么" not in query: warning = f"{Colors.WARNING}[跨语言风险]{Colors.ENDC}"
|
||
|
|
|
||
|
|
print(f" {color}✅ 命中 ({rank_display}){Colors.ENDC} | MRR: {reciprocal_rank:.2f} | 耗时: {latency:.0f}ms {warning}")
|
||
|
|
else:
|
||
|
|
print(f" {Colors.FAIL}❌ 未命中{Colors.ENDC} | 预期关键词: {item['keywords']}")
|
||
|
|
|
||
|
|
# 稍微间隔，避免触发 API 频率限制
|
||
|
|
sleep(0.1)
|
||
|
|
|
||
|
|
# === 最终计算 ===
|
||
|
|
count = len(dataset)
|
||
|
|
if count == 0: return
|
||
|
|
|
||
|
|
avg_p1 = np.mean(metrics['p_at_1']) * 100
|
||
|
|
avg_hit5 = np.mean(metrics['hit_at_5']) * 100
|
||
|
|
avg_mrr = np.mean(metrics['mrr'])
|
||
|
|
avg_latency = np.mean(metrics['latency'])
|
||
|
|
p95_latency = np.percentile(metrics['latency'], 95)
|
||
|
|
|
||
|
|
print("\n" + "="*60)
|
||
|
|
print(f"{Colors.HEADER}📊 最终量化评估报告 (Evaluation Report){Colors.ENDC}")
|
||
|
|
print("="*60)
|
||
|
|
|
||
|
|
# 1. Precision@1 (最关键指标)
|
||
|
|
print(f"🥇 {Colors.BOLD}Precision@1 (首位精确率): {avg_p1:.1f}%{Colors.ENDC}")
|
||
|
|
print(f" - 意义: 用户能否直接得到正确答案。引入 Rerank 后此项应显著提高。")
|
||
|
|
|
||
|
|
# 2. Hit Rate / Recall@5
|
||
|
|
print(f"🥈 Hit Rate@5 (前五召回率): {avg_hit5:.1f}%")
|
||
|
|
print(f" - 意义: 数据库是否真的包含答案。如果此项低，说明爬虫没爬全或混合检索漏了。")
|
||
|
|
|
||
|
|
# 3. MRR
|
||
|
|
print(f"🥉 MRR (平均倒数排名): {avg_mrr:.3f} / 1.0")
|
||
|
|
|
||
|
|
# 4. Latency
|
||
|
|
print(f"⚡ Avg Latency (平均耗时): {avg_latency:.0f} ms")
|
||
|
|
print(f"⚡ P95 Latency (95%分位): {p95_latency:.0f} ms")
|
||
|
|
print("="*60)
|
||
|
|
|
||
|
|
# === 智能诊断 ===
|
||
|
|
print(f"{Colors.HEADER}💡 诊断建议:{Colors.ENDC}")
|
||
|
|
|
||
|
|
if avg_p1 < avg_hit5:
|
||
|
|
gap = avg_hit5 - avg_p1
|
||
|
|
print(f" • {Colors.WARNING}排序优化空间大{Colors.ENDC}: 召回了但没排第一的情况占 {gap:.1f}%。")
|
||
|
|
print(" -> 你的 Rerank 模型生效了吗？或者 Rerank 的 Top N 截断是否太早？")
|
||
|
|
elif avg_p1 > 80:
|
||
|
|
print(f" • {Colors.OKGREEN}排序效果优秀{Colors.ENDC}: 绝大多数正确答案都排在第一位。")
|
||
|
|
|
||
|
|
if avg_hit5 < 50:
|
||
|
|
print(f" • {Colors.FAIL}召回率过低{Colors.ENDC}: 可能是测试集关键词太生僻，或者 TS_RANK 权重过低。")
|
||
|
|
|
||
|
|
if avg_latency > 2000:
|
||
|
|
print(f" • {Colors.WARNING}系统响应慢{Colors.ENDC}: 2秒以上。检查是否因为 Rerank 文档过多(建议 <= 50个)。")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
run_evaluation()
|