146 lines
5.2 KiB
Python
146 lines
5.2 KiB
Python
|
"""
|
|||
|
|
因为map_json和history_json的传入格式不同，在main里改太麻烦
|
|||
|
|
为了解耦，所以写两个函数将map和history转换为标准结构
|
|||
|
|
urls = [
|
|||
|
|
"https://www.baidu.com",
|
|||
|
|
"https://www.taobao.com",
|
|||
|
|
"https://www.jd.com",
|
|||
|
|
"https://www.1688.com",
|
|||
|
|
"https://www.taobao.com",
|
|||
|
|
"https://www.jd.com",
|
|||
|
|
"https://www.1688.com",
|
|||
|
|
]
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def map_json_transform(map_json: list[dict]) -> list[dict]:
|
|||
|
|
"""
|
|||
|
|
将Map节点的输出转为干净的输出，避免杂七杂八的数据干扰
|
|||
|
|
输入: Map节点的输出
|
|||
|
|
输出: 转换后的 JSON 数组，每个元素包含 title, url, markdown 字段
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
# 如果 Dify 传入的是字符串，先转为字典
|
|||
|
|
map_obj = json.loads(map_json) if isinstance(map_json, str) else map_json
|
|||
|
|
# map_obj首先被列表包裹
|
|||
|
|
map_obj = map_obj[0]
|
|||
|
|
# 直接从json里提取 links 数组
|
|||
|
|
if isinstance(map_obj, dict) and "links" in map_obj:
|
|||
|
|
links = map_obj["links"]
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
return {"targets": [], "msg": f"Map数据解析失败: {str(e)}"}
|
|||
|
|
|
|||
|
|
return {"targets": links, "msg": "Map数据解析成功"}
|
|||
|
|
|
|||
|
|
def history_json_transform(history_json: list[dict]) -> list[dict]:
|
|||
|
|
"""
|
|||
|
|
将History节点的输出转为干净的输出，避免杂七杂八的数据干扰
|
|||
|
|
输入: History节点的输出
|
|||
|
|
输出: 转换后的 JSON 数组，每个元素包含 url 字段
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
# 如果 Dify 传入的是字符串，先转为字典
|
|||
|
|
hist_obj = json.loads(history_json) if isinstance(history_json, str) else history_json
|
|||
|
|
# hist_obj首先被列表包裹
|
|||
|
|
hist_obj = hist_obj[0]
|
|||
|
|
# 将data解析出来
|
|||
|
|
hist_obj = hist_obj['data'][0]
|
|||
|
|
# 得到result
|
|||
|
|
|
|||
|
|
# 直接从json里提取 hist_data 数组
|
|||
|
|
if isinstance(hist_obj, dict) and "data" in hist_obj:
|
|||
|
|
hist_data = hist_obj["data"]
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
return {"targets": [], "msg": f"History数据解析失败: {str(e)}"}
|
|||
|
|
|
|||
|
|
return {"targets": hist_data, "msg": "History数据解析成功"}
|
|||
|
|
|
|||
|
|
def main(map_json: list[dict], history_json: list[dict], batch_size: float):
|
|||
|
|
"""
|
|||
|
|
map_json: Firecrawl Map 节点的输出
|
|||
|
|
history_json: 数据库查询节点的输出 (包含 hist_data 数组)
|
|||
|
|
batch_size: 每次处理的数量
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
# 1. 解析 Map 数据 (全量链接)
|
|||
|
|
all_links = []
|
|||
|
|
try:
|
|||
|
|
# 如果 Dify 传入的是字符串，先转为字典
|
|||
|
|
map_obj = json.loads(map_json) if isinstance(map_json, str) else map_json
|
|||
|
|
# map_obj首先被列表包裹
|
|||
|
|
map_obj = map_obj[0]
|
|||
|
|
# 直接从json里提取 links 数组
|
|||
|
|
if isinstance(map_obj, dict) and "links" in map_obj:
|
|||
|
|
all_links = map_obj["links"]
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
return {"targets": [], "msg": f"Map数据解析失败: {str(e)}"}
|
|||
|
|
|
|||
|
|
# 2. 解析 History 数据 (已完成链接)
|
|||
|
|
completed_set = set()
|
|||
|
|
try:
|
|||
|
|
hist_obj = json.loads(history_json) if isinstance(history_json, str) else history_json
|
|||
|
|
# hist_obj首先被列表包裹
|
|||
|
|
hist_obj = hist_obj[0]
|
|||
|
|
# 直接从json里提取 hist_data 数组
|
|||
|
|
if isinstance(hist_obj, dict) and "data" in hist_obj:
|
|||
|
|
hist_data = hist_obj["data"]
|
|||
|
|
|
|||
|
|
# 将 hist_data 里的 url 提取出来放入集合 (Set) 做 O(1) 查找
|
|||
|
|
for item in hist_data:
|
|||
|
|
if isinstance(item, dict):
|
|||
|
|
url_val = item.get("url")
|
|||
|
|
if url_val:
|
|||
|
|
completed_set.add(url_val)
|
|||
|
|
|
|||
|
|
except Exception as e:
|
|||
|
|
return {"targets": [], "msg": f"History数据解析失败: {str(e)}"}
|
|||
|
|
|
|||
|
|
# 3. 核心逻辑: 过滤 (Diff)
|
|||
|
|
targets = []
|
|||
|
|
for link in all_links:
|
|||
|
|
# A. 基础校验: 必须是字符串且以 http 开头
|
|||
|
|
if not isinstance(link, str) or not link.startswith("http"):
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# B. 过滤 sitemap.xml 自身
|
|||
|
|
if link.endswith(".xml") or "sitemap" in link.split("/")[-1]:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# C. 过滤已爬取的 (关键步骤)
|
|||
|
|
if link in completed_set:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
targets.append(link)
|
|||
|
|
|
|||
|
|
# 4. 分批 (Batch)
|
|||
|
|
try:
|
|||
|
|
limit = int(batch_size)
|
|||
|
|
except:
|
|||
|
|
limit = 10 # 默认值
|
|||
|
|
|
|||
|
|
current_batch = targets[:limit]
|
|||
|
|
remaining_count = len(targets) - len(current_batch)
|
|||
|
|
|
|||
|
|
# 5. 返回结果
|
|||
|
|
return {
|
|||
|
|
"targets": current_batch,
|
|||
|
|
"count_urls": len(all_links),
|
|||
|
|
"count_completed": len(completed_set),
|
|||
|
|
"count_remaining": remaining_count,
|
|||
|
|
"count_error": 0
|
|||
|
|
# "msg": f"全站发现: {len(all_links)} | 已入库: {len(completed_set)} | 本次执行: {len(current_batch)} | 剩余待爬: {remaining_count}"
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
import json
|
|||
|
|
with open("anyscript\wiki_crawler\diff.json", "r", encoding="utf-8") as f:
|
|||
|
|
data = json.load(f)
|
|||
|
|
map_json = data["map_json"]
|
|||
|
|
history_json = data["history_json"]
|
|||
|
|
batch_size = data["batch_size"]
|
|||
|
|
|
|||
|
|
result = main(map_json, history_json, batch_size)
|
|||
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|