修改地图爬取urls，应对dify的列表上限和http超时

backend/main.py

@@ -16,7 +16,7 @@ async def register(req: RegisterRequest):
 @app.post("/add_urls")
 async def add_urls(req: AddUrlsRequest):
     try:
-        data = crawler_service.add_urls(req.task_id, req.urls)
+        data = crawler_service.add_urls(req.task_id, req.urls_obj)
         return make_response(1, "Success", data)
     except Exception as e:
         return make_response(0, str(e))

backend/schemas.py

@@ -10,7 +10,7 @@ class PendingRequest(BaseModel):
 
 class AddUrlsRequest(BaseModel):
     task_id: int
-    urls: List[str]
+    urls_obj: dict
 
 # schemas.py
 class CrawlResult(BaseModel):

backend/service.py

@@ -23,15 +23,17 @@ class CrawlerService:
             new_task = conn.execute(stmt).fetchone()
             return {"task_id": new_task[0], "is_new_task": True}
 
-    def add_urls(self, task_id: int, urls: list):
+    def add_urls(self, task_id: int, urls_obj: dict):
         """通用 API 实现的批量添加（含详细返回）"""
         success_urls, skipped_urls, failed_urls = [], [], []
-        
+        # 从 urls_obj 中提取 urls 列表
+        urls = urls_obj.get("urls", [])
+
         with self.db.engine.begin() as conn:
             for url in urls:
                 clean_url = normalize_url(url)
                 try:
-                    # 检查是否存在 (通用写法)
+                    # 检查队列中是否已存在该 URL (通用写法)
                     check_q = select(self.db.queue).where(
                         and_(self.db.queue.c.task_id == task_id, self.db.queue.c.url == clean_url)
                     )