106 lines
3.2 KiB
Python
106 lines
3.2 KiB
Python
import json
|
||
from urllib.parse import urlparse, urlunparse
|
||
from langchain_community.utilities import SQLDatabase
|
||
from sqlalchemy import Table, MetaData, select, insert, update, delete, and_
|
||
|
||
# --- 工具函数：URL 标准化 ---
|
||
def normalize_url(url: str) -> str:
|
||
"""
|
||
标准化 URL，确保末尾斜杠、大小写等不影响唯一性判定
|
||
"""
|
||
if not url:
|
||
return url
|
||
|
||
# 1. 解析 URL
|
||
parsed = urlparse(url.strip())
|
||
|
||
# 2. 转换协议和域名为小写 (Domain 是不区分大小写的)
|
||
scheme = parsed.scheme.lower()
|
||
netloc = parsed.netloc.lower()
|
||
|
||
# 3. 处理路径：去除末尾的斜杠
|
||
path = parsed.path
|
||
if path.endswith('/'):
|
||
path = path.rstrip('/')
|
||
|
||
# 4. 去除 Fragment (#部分)，保留 Query 参数
|
||
# 如果需要忽略 Query 参数，可以将 query 设置为 ""
|
||
query = parsed.query
|
||
|
||
# 5. 重新拼接
|
||
normalized = urlunparse((scheme, netloc, path, parsed.params, query, ""))
|
||
return normalized
|
||
|
||
# --- 数据库连接工厂 ---
|
||
def get_db_connection(db_url: str):
|
||
"""
|
||
获取通用数据库连接，处理协议兼容性
|
||
"""
|
||
if db_url.startswith("postgres://"):
|
||
db_url = db_url.replace("postgres://", "postgresql+psycopg2://", 1)
|
||
elif db_url.startswith("postgresql://") and "+psycopg2" not in db_url:
|
||
db_url = db_url.replace("postgresql://", "postgresql+psycopg2://", 1)
|
||
|
||
try:
|
||
# engine_args 确保连接池在 Dify 高并发下更稳定
|
||
return SQLDatabase.from_uri(db_url, engine_args={
|
||
"pool_pre_ping": True,
|
||
"pool_recycle": 3600
|
||
})
|
||
except Exception as e:
|
||
raise RuntimeError(f"DB_CONNECT_ERROR: {str(e)}")
|
||
|
||
# --- Dify 节点主入口 ---
|
||
def main(inputs: dict):
|
||
"""
|
||
Dify 节点主入口函数
|
||
"""
|
||
ret = {"code": 0, "msg": "unknown", "data": None}
|
||
|
||
# 预设数据库连接字符串 (建议在 Dify 环境变量中配置)
|
||
db_url = inputs.get("db_url")
|
||
|
||
try:
|
||
# 1. 初始化数据库
|
||
db = get_db_connection(db_url)
|
||
|
||
# 2. 执行具体的业务逻辑
|
||
result_data = _logic_handler(db, inputs)
|
||
|
||
ret["code"] = 1
|
||
ret["msg"] = "success"
|
||
ret["data"] = result_data
|
||
|
||
except Exception as e:
|
||
ret["code"] = 0
|
||
ret["msg"] = str(e)
|
||
ret["data"] = None
|
||
|
||
return ret
|
||
|
||
# -------------------------------------------------
|
||
# 业务逻辑处理器：每个节点只需修改这里
|
||
# -------------------------------------------------
|
||
def _logic_handler(db: SQLDatabase, inputs: dict):
|
||
"""
|
||
在这里编写具体的业务操作
|
||
"""
|
||
engine = db._engine
|
||
metadata = MetaData()
|
||
|
||
# 示例：获取并标准化 URL
|
||
raw_url = inputs.get("url", "")
|
||
clean_url = normalize_url(raw_url)
|
||
|
||
# 反射获取表对象
|
||
# tasks = Table('crawl_tasks', metadata, autoload_with=engine)
|
||
|
||
# 使用 SQLAlchemy Core 进行操作（无需写原生SQL）
|
||
# with engine.begin() as conn:
|
||
# stmt = select(tasks).where(tasks.c.root_url == clean_url)
|
||
# result = conn.execute(stmt).fetchone()
|
||
|
||
return {
|
||
"processed_url": clean_url,
|
||
"info": "逻辑已执行"
|
||
} |