v3接口restful风格，规范化接口；添加mcp服务器；新增log模块

backend/services/crawler_service.py

@@ -1,182 +1,179 @@
-# backend/services/crawler_service.py
+import concurrent.futures
+import threading
 from firecrawl import FirecrawlApp
 from backend.core.config import settings
 from backend.services.data_service import data_service
 from backend.services.llm_service import llm_service
 from backend.utils.text_process import text_processor
+import logging
 
+# 获取当前模块的专用 Logger
+# __name__ 会自动识别为 "backend.services.crawler_service" 这样的路径
+logger = logging.getLogger(__name__)
 class CrawlerService:
-    """
-    爬虫编排服务
-    协调 Firecrawl, LLM, 和 DataService
-    """
     def __init__(self):
         self.firecrawl = FirecrawlApp(api_key=settings.FIRECRAWL_API_KEY)
+        self.max_workers = 5 
+        
+        # [新增] 内存状态追踪
+        # 结构: { task_id: { url: "status_desc" } }
+        self._active_workers = {} 
+        self._lock = threading.Lock()
+
+    def _track_start(self, task_id, url):
+        """开始追踪某个URL"""
+        with self._lock:
+            if task_id not in self._active_workers:
+                self._active_workers[task_id] = set()
+            self._active_workers[task_id].add(url)
+
+    def _track_end(self, task_id, url):
+        """结束追踪某个URL"""
+        with self._lock:
+            if task_id in self._active_workers:
+                self._active_workers[task_id].discard(url)
+
+    def get_task_status(self, task_id: int):
+        """
+        [综合监控] 获取全量状态 = 数据库统计 + 实时线程列表
+        """
+        # 1. 获取数据库层面的统计 (宏观)
+        db_data = data_service.get_task_monitor_data(task_id)
+        if not db_data:
+            return None
+
+        # 2. 获取内存层面的活跃线程 (微观)
+        with self._lock:
+            active_urls = list(self._active_workers.get(task_id, []))
+        # 输出情况
+        logger.info(f"Task {task_id} active threads: {active_urls}")
+        logger.info(f"Task {task_id} stats: {db_data['db_stats']}") # 打印数据库统计信息
+
+        return {
+            "root_url": db_data["root_url"],
+            "stats": db_data["db_stats"],       # Pending, Completed, Failed 等
+            "active_threads": active_urls,      # 当前 CPU/网络 正在处理的 URL
+            "active_thread_count": len(active_urls)
+        }
 
     def map_site(self, start_url: str):
-        print(f"[INFO] Mapping: {start_url}")
+        """阶段1：站点地图扫描"""
+        logger.info(f"Mapping: {start_url}")
         try:
-            # 1. 注册任务
             task_res = data_service.register_task(start_url)
+            urls_to_add = [start_url] 
             
-            # 2. 无论是否新任务，都尝试把 start_url 加入队列
-            urls_to_add = [start_url]
-
-            # 3. 调用 Firecrawl Map
+            # 如果任务已存在，不再重新 Map，直接返回
+            if not task_res['is_new_task']:
+                logger.info(f"Task {task_res['task_id']} exists, skipping map.")
+                return {
+                    "task_id": task_res['task_id'], 
+                    "count": 0,
+                    "is_new": False
+                }
+            
+            # 新任务执行 Map
             try:
                 map_res = self.firecrawl.map(start_url)
-                
-                # 兼容不同 SDK 版本的返回 (对象 或 字典)
-                found_links = []
-                if isinstance(map_res, dict):
-                    found_links = map_res.get('links', [])
-                else:
-                    found_links = getattr(map_res, 'links', [])
-                
+                found_links = map_res.get('links', []) if isinstance(map_res, dict) else getattr(map_res, 'links', [])
                 for link in found_links:
-                    # 链接可能是字符串，也可能是对象
                     u = link if isinstance(link, str) else getattr(link, 'url', str(link))
                     urls_to_add.append(u)
-                    
-                print(f"[INFO] Firecrawl Map found {len(found_links)} sub-links.")
+                logger.info(f"Map found {len(found_links)} links")
             except Exception as e:
-                print(f"[WARN] Firecrawl Map warning (proceeding with seed only): {e}")
+                logger.warning(f"Map failed, proceeding with seed only: {e}")
 
-            # 4. 批量入库
             if urls_to_add:
                 data_service.add_urls(task_res['task_id'], urls_to_add)
             
             return {
-                "msg": "Mapped successfully", 
-                "count": len(urls_to_add), 
-                "task_id": task_res['task_id']
+                "task_id": task_res['task_id'], 
+                "count": len(urls_to_add),
+                "is_new": True
             }
-
         except Exception as e:
-            print(f"[ERROR] Map failed: {e}")
+            logger.error(f"Map failed: {e}")
             raise e
 
-    def process_queue(self, task_id: int, batch_size: int = 5):
-        """
-        处理队列：爬取 -> 清洗 -> 切分 -> 向量化 -> 存储
-        """
-        pending = data_service.get_pending_urls(task_id, batch_size)
-        urls = pending['urls']
-        if not urls: return {"msg": "Queue empty"}
-
-        processed = 0
-        for url in urls:
-            try:
-                print(f"[INFO] Scraping: {url}")
-                
-                # 调用 Firecrawl
-                scrape_res = self.firecrawl.scrape(
-                    url, 
-                    formats=['markdown'], 
-                    only_main_content=True
-                )
-                
-                # =====================================================
-                # 【核心修复】兼容字典和对象两种返回格式
-                # =====================================================
-                raw_md = ""
-                metadata = None
-
-                # 1. 提取 markdown 和 metadata 本体
-                if isinstance(scrape_res, dict):
-                    raw_md = scrape_res.get('markdown', '')
-                    metadata = scrape_res.get('metadata', {})
-                else:
-                    # 如果是对象 (ScrapeResponse)
-                    raw_md = getattr(scrape_res, 'markdown', '')
-                    metadata = getattr(scrape_res, 'metadata', {})
-
-                # 2. 从 metadata 中安全提取 title
-                title = url # 默认用 url 当标题
-                if metadata:
-                    if isinstance(metadata, dict):
-                        # 如果 metadata 是字典
-                        title = metadata.get('title', url)
-                    else:
-                        # 如果 metadata 是对象 (DocumentMetadata) -> 这里就是你报错的地方
-                        title = getattr(metadata, 'title', url)
-                
-                # =====================================================
-
-                if not raw_md:
-                    print(f"[WARN] Empty content for {url}")
-                    continue
-
-                # 1. 清洗
-                clean_md = text_processor.clean_markdown(raw_md)
-                if not clean_md: continue
-
-                # 2. 切分
-                chunks = text_processor.split_markdown(clean_md)
-                
-                chunks_data = []
-                for i, chunk in enumerate(chunks):
-                    content = chunk['content']
-                    headers = chunk['metadata']
-                    header_path = " > ".join(headers.values())
-                    
-                    # 3. 向量化 (Title + Header + Content)
-                    emb_input = f"{title}\n{header_path}\n{content}"
-                    vector = llm_service.get_embedding(emb_input)
-                    
-                    if vector:
-                        chunks_data.append({
-                            "index": i,
-                            "content": content,
-                            "embedding": vector,
-                            "meta_info": {"header_path": header_path, "headers": headers}
-                        })
-                
-                # 4. 存储
-                if chunks_data:
-                    data_service.save_chunks(task_id, url, title, chunks_data)
-                    processed += 1
-            except Exception as e:
-                # 打印详细错误方便调试
-                print(f"[ERROR] Process URL {url} failed: {e}")
+    def _process_single_url(self, task_id: int, url: str):
+        """[Worker] 单个 URL 处理线程"""
+        # 1. 内存标记：开始
+        self._track_start(task_id, url)
+        logger.info(f"[THREAD START] {url}")
         
-        return {"msg": "Batch processed", "count": processed}
+        try:
+            # 2. 爬取
+            scrape_res = self.firecrawl.scrape(
+                url, formats=['markdown'], only_main_content=True
+            )
+            
+            raw_md = getattr(scrape_res, 'markdown', '') if not isinstance(scrape_res, dict) else scrape_res.get('markdown', '')
+            metadata = getattr(scrape_res, 'metadata', {}) if not isinstance(scrape_res, dict) else scrape_res.get('metadata', {})
+            title = getattr(metadata, 'title', url) if not isinstance(metadata, dict) else metadata.get('title', url)
+
+            if not raw_md:
+                data_service.mark_url_status(task_id, url, 'failed')
+                return
+
+            # 3. 清洗 & 切分
+            clean_md = text_processor.clean_markdown(raw_md)
+            chunks = text_processor.split_markdown(clean_md)
+            
+            chunks_data = []
+            for i, chunk in enumerate(chunks):
+                headers = chunk['metadata']
+                path = " > ".join(headers.values())
+                emb_input = f"{title}\n{path}\n{chunk['content']}"
+                vector = llm_service.get_embedding(emb_input)
+                if vector:
+                    chunks_data.append({
+                        "index": i, "content": chunk['content'], "embedding": vector,
+                        "meta_info": {"header_path": path, "headers": headers}
+                    })
+            
+            # 4. 入库 (会自动标记 completed)
+            if chunks_data:
+                data_service.save_chunks(task_id, url, title, chunks_data)
+            else:
+                data_service.mark_url_status(task_id, url, 'failed')
+
+        except Exception as e:
+            logger.error(f"[THREAD ERROR] {url}: {e}")
+            data_service.mark_url_status(task_id, url, 'failed')
+        
+        finally:
+            # 5. 内存标记：结束 (无论成功失败都要移除)
+            self._track_end(task_id, url)
+
+    def process_queue_concurrent(self, task_id: int, batch_size: int = 10):
+        """阶段2：多线程并发处理"""
+        urls = data_service.get_pending_urls(task_id, limit=batch_size)
+        if not urls: return {"msg": "No pending urls"}
+
+        logger.info(f"Batch started: {len(urls)} urls")
+        
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # 提交任务到线程池
+            futures = {executor.submit(self._process_single_url, task_id, url): url for url in urls}
+            # 等待完成
+            concurrent.futures.wait(futures)
+
+        return {"msg": "Batch completed", "count": len(urls)}
 
     def search(self, query: str, task_id, return_num: int):
-        """
-        全链路搜索：向量生成 -> 混合检索(粗排) -> 重排序(精排)
-        """
-        # 1. 生成查询向量
+        """阶段3：搜索"""
         vector = llm_service.get_embedding(query)
         if not vector: return {"msg": "Embedding failed", "results": []}
 
-        # 2. 计算粗排召回数量
-        # 逻辑：至少召回 50 个，如果用户要很多，则召回 10 倍
-        coarse_limit = return_num * 10 if return_num * 10 > settings.CANDIDATE_NUM else settings.CANDIDATE_NUM
+        coarse_limit = min(return_num * 10, 100)
+        coarse_limit = max(coarse_limit, 50)
         
-        # 3. 执行混合检索 (粗排)
-        coarse_results = data_service.search(
-            query_text=query, 
-            query_vector=vector, 
-            task_id=task_id, 
-            candidates_num=coarse_limit # 使用计算出的粗排数量
-        )
-        
-        candidates = coarse_results.get('results', [])
-        
-        if not candidates:
-            return {"msg": "No documents found", "results": []}
+        coarse_res = data_service.search(query, vector, task_id, coarse_limit)
+        candidates = coarse_res.get('results', [])
 
-        # 4. 执行重排序 (精排)
-        final_results = llm_service.rerank(
-            query=query, 
-            documents=candidates, 
-            top_n=return_num # 最终返回用户需要的数量
-        )
+        if not candidates: return {"results": []}
+
+        final_res = llm_service.rerank(query, candidates, return_num)
+        return {"results": final_res}
 
-        return {
-            "results": final_results, 
-            "msg": f"Reranked {len(final_results)} from {len(candidates)} candidates"
-        }
-    
 crawler_service = CrawlerService()

backend/services/data_service.py

@@ -1,11 +1,14 @@
 from sqlalchemy import select, insert, update, and_, text, func, desc
 from backend.core.database import db
 from backend.utils.common import normalize_url
+import logging
 
+# 获取当前模块的专用 Logger
+# __name__ 会自动识别为 "backend.services.crawler_service" 这样的路径
+logger = logging.getLogger(__name__)
 class DataService:
     """
     数据持久化服务层
-    只负责数据库 CRUD 操作，不包含外部 API 调用逻辑
     """
     def __init__(self):
         self.db = db
@@ -17,11 +20,11 @@ class DataService:
             existing = conn.execute(query).fetchone()
             
             if existing:
-                return {"task_id": existing[0], "is_new_task": False, "msg": "Task already exists"}
+                return {"task_id": existing[0], "is_new_task": False}
             else:
                 stmt = insert(self.db.tasks).values(root_url=clean_url).returning(self.db.tasks.c.id)
                 new_task = conn.execute(stmt).fetchone()
-                return {"task_id": new_task[0], "is_new_task": True, "msg": "New task created"}
+                return {"task_id": new_task[0], "is_new_task": True}
 
     def add_urls(self, task_id: int, urls: list[str]):
         success_urls = []
@@ -29,7 +32,7 @@ class DataService:
             for url in urls:
                 clean_url = normalize_url(url)
                 try:
-                    check_q = select(self.db.queue).where(
+                    check_q = select(self.db.queue.c.id).where(
                         and_(self.db.queue.c.task_id == task_id, self.db.queue.c.url == clean_url)
                     )
                     if not conn.execute(check_q).fetchone():
@@ -41,115 +44,111 @@ class DataService:
 
     def get_pending_urls(self, task_id: int, limit: int):
         with self.db.engine.begin() as conn:
-            query = select(self.db.queue.c.url).where(
+            # 原子锁定：获取并标记为 processing
+            subquery = select(self.db.queue.c.id).where(
                 and_(self.db.queue.c.task_id == task_id, self.db.queue.c.status == 'pending')
-            ).limit(limit)
-            urls = [r[0] for r in conn.execute(query).fetchall()]
+            ).limit(limit).with_for_update(skip_locked=True)
             
-            if urls:
-                conn.execute(update(self.db.queue).where(
-                    and_(self.db.queue.c.task_id == task_id, self.db.queue.c.url.in_(urls))
-                ).values(status='processing'))
-        
-        return {"urls": urls, "msg": "Fetched pending urls"}
-        
+            stmt = update(self.db.queue).where(
+                self.db.queue.c.id.in_(subquery)
+            ).values(status='processing').returning(self.db.queue.c.url)
+            
+            result = conn.execute(stmt).fetchall()
+            return [r[0] for r in result]
+
+    def mark_url_status(self, task_id: int, url: str, status: str):
+        clean_url = normalize_url(url)
+        with self.db.engine.begin() as conn:
+            conn.execute(update(self.db.queue).where(
+                and_(self.db.queue.c.task_id == task_id, self.db.queue.c.url == clean_url)
+            ).values(status=status))
+
+    def get_task_monitor_data(self, task_id: int):
+        """[数据库层监控] 获取持久化的任务状态"""
+        with self.db.engine.connect() as conn:
+            # 1. 检查任务是否存在
+            task_exists = conn.execute(select(self.db.tasks.c.root_url).where(self.db.tasks.c.id == task_id)).fetchone()
+            if not task_exists:
+                return None
+
+            # 2. 统计各状态数量
+            stats_rows = conn.execute(select(
+                self.db.queue.c.status, func.count(self.db.queue.c.id)
+            ).where(self.db.queue.c.task_id == task_id).group_by(self.db.queue.c.status)).fetchall()
+            
+            stats = {"pending": 0, "processing": 0, "completed": 0, "failed": 0}
+            for status, count in stats_rows:
+                if status in stats: stats[status] = count
+            stats["total"] = sum(stats.values())
+            
+            return {
+                "root_url": task_exists[0],
+                "db_stats": stats
+            }
+
     def save_chunks(self, task_id: int, source_url: str, title: str, chunks_data: list):
-        """
-        保存切片数据 (Phase 1.5: 支持 meta_info)
-        """
         clean_url = normalize_url(source_url)
-        count = 0
         with self.db.engine.begin() as conn:
             for item in chunks_data:
-                # item 结构: {'index': int, 'content': str, 'embedding': list, 'meta_info': dict}
                 idx = item['index']
                 meta = item.get('meta_info', {})
-                
-                # 检查是否存在
                 existing = conn.execute(select(self.db.chunks.c.id).where(
                     and_(self.db.chunks.c.task_id == task_id, 
                          self.db.chunks.c.source_url == clean_url,
                          self.db.chunks.c.chunk_index == idx)
                 )).fetchone()
-
                 values = {
                     "task_id": task_id, "source_url": clean_url, "chunk_index": idx,
                     "title": title, "content": item['content'], "embedding": item['embedding'],
                     "meta_info": meta
                 }
-
                 if existing:
                     conn.execute(update(self.db.chunks).where(self.db.chunks.c.id == existing[0]).values(**values))
                 else:
                     conn.execute(insert(self.db.chunks).values(**values))
-                count += 1
             
-            # 标记队列完成
             conn.execute(update(self.db.queue).where(
                 and_(self.db.queue.c.task_id == task_id, self.db.queue.c.url == clean_url)
             ).values(status='completed'))
 
-        return {"msg": f"Saved {count} chunks", "count": count}
-
-    def search(self, query_text: str, query_vector: list, task_id=None, candidates_num: int = 5):
-        """
-        Phase 2: 混合检索 (Hybrid Search)
-        """
+    def search(self, query_text: str, query_vector: list, task_id=None, candidates_num: int = 50):
         # 向量格式清洗
         if hasattr(query_vector, 'tolist'): query_vector = query_vector.tolist()
-        if query_vector and isinstance(query_vector, list) and len(query_vector) > 0:
-            if isinstance(query_vector[0], list): query_vector = query_vector[0]
+        if isinstance(query_vector, list) and len(query_vector) > 0 and isinstance(query_vector[0], list):
+            query_vector = query_vector[0]
 
         results = []
         with self.db.engine.connect() as conn:
-            keyword_query = func.websearch_to_tsquery('english', query_text) # 转换为 tsquery
-            vector_score = (1 - self.db.chunks.c.embedding.cosine_distance(query_vector))# 计算向量相似度
-            keyword_score = func.ts_rank(self.db.chunks.c.content_tsvector, keyword_query) # 计算关键词相似度
-            final_score = (vector_score * 0.7 + func.coalesce(keyword_score, 0) * 0.3).label("score")# 计算最终分数
+            keyword_query = func.websearch_to_tsquery('english', query_text)
+            vector_score = (1 - self.db.chunks.c.embedding.cosine_distance(query_vector))
+            keyword_score = func.ts_rank(self.db.chunks.c.content_tsvector, keyword_query)
+            final_score = (vector_score * 0.7 + func.coalesce(keyword_score, 0) * 0.3).label("score")
 
             stmt = select(
-                self.db.chunks.c.task_id,
-                self.db.chunks.c.source_url,
-                self.db.chunks.c.title,
-                self.db.chunks.c.content,
-                self.db.chunks.c.meta_info,
-                final_score
+                self.db.chunks.c.task_id, self.db.chunks.c.source_url, self.db.chunks.c.title,
+                self.db.chunks.c.content, self.db.chunks.c.meta_info, final_score
             )
-
-            if task_id:
-                stmt = stmt.where(self.db.chunks.c.task_id == task_id)
-            
-            # 使用 candidates_num 控制召回数量
+            if task_id: stmt = stmt.where(self.db.chunks.c.task_id == task_id)
             stmt = stmt.order_by(desc("score")).limit(candidates_num)
             
             try:
                 rows = conn.execute(stmt).fetchall()
-                results = [
-                    {
-                        "task_id": r[0], 
-                        "source_url": r[1], 
-                        "title": r[2], 
-                        "content": r[3], 
-                        "meta_info": r[4],
-                        "score": float(r[5])
-                    } 
-                    for r in rows
-                ]
+                results = [{"task_id": r[0], "source_url": r[1], "title": r[2], "content": r[3], "meta_info": r[4], "score": float(r[5])} for r in rows]
             except Exception as e:
-                print(f"[ERROR] Hybrid search failed: {e}")
+                logger.error(f"Hybrid search failed: {e}")
                 return self._fallback_vector_search(query_vector, task_id, candidates_num)
 
         return {"results": results, "msg": f"Hybrid found {len(results)}"}
 
     def _fallback_vector_search(self, vector, task_id, limit):
-        print("[WARN] Fallback to pure vector search")
+        logger.warning("Fallback to pure vector search")
         with self.db.engine.connect() as conn:
             stmt = select(
                 self.db.chunks.c.task_id, self.db.chunks.c.source_url, self.db.chunks.c.title,
                 self.db.chunks.c.content, self.db.chunks.c.meta_info
             ).order_by(self.db.chunks.c.embedding.cosine_distance(vector)).limit(limit)
-            if task_id:
-                stmt = stmt.where(self.db.chunks.c.task_id == task_id)
+            if task_id: stmt = stmt.where(self.db.chunks.c.task_id == task_id)
             rows = conn.execute(stmt).fetchall()
-            return {"results": [{"content": r[3], "meta_info": r[4]} for r in rows], "msg": "Fallback found"}
+            return {"results": [{"content": r[3], "meta_info": r[4], "score": 0.0} for r in rows], "msg": "Fallback found"}
+
 data_service = DataService()

backend/services/llm_service.py

@@ -1,7 +1,11 @@
 import dashscope
 from http import HTTPStatus
 from backend.core.config import settings
+import logging
 
+# 获取当前模块的专用 Logger
+# __name__ 会自动识别为 "backend.services.crawler_service" 这样的路径
+logger = logging.getLogger(__name__)
 class LLMService:
     """
     LLM 服务封装层
@@ -21,10 +25,10 @@ class LLMService:
             if resp.status_code == HTTPStatus.OK:
                 return resp.output['embeddings'][0]['embedding']
             else:
-                print(f"[ERROR] Embedding API Error: {resp}")
+                logger.error(f"Embedding API Error: {resp}")
                 return None
         except Exception as e:
-            print(f"[ERROR] Embedding Exception: {e}")
+            logger.error(f"Embedding Exception: {e}")
             return None
 
     def rerank(self, query: str, documents: list, top_n: int = 5):
@@ -81,12 +85,12 @@ class LLMService:
                 
                 return reranked_results
             else:
-                print(f"[ERROR] Rerank API Error: {resp}")
+                logger.error(f"Rerank API Error: {resp}")
                 # 降级策略：如果 Rerank 挂了，直接返回粗排的前 N 个
                 return documents[:top_n]
                 
         except Exception as e:
-            print(f"[ERROR] Rerank Exception: {e}")
+            logger.error(f"Rerank Exception: {e}")
             # 降级策略
             return documents[:top_n]