61 lines
2.1 KiB
Python
61 lines
2.1 KiB
Python
|
import re
|
|||
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownHeaderTextSplitter
|
|||
|
|
|
|||
|
|
class TextProcessor:
|
|||
|
|
"""文本处理工具类：负责 Markdown 清洗和切分"""
|
|||
|
|
|
|||
|
|
def __init__(self):
|
|||
|
|
# 基于 Markdown 标题的语义切分器
|
|||
|
|
self.md_splitter = MarkdownHeaderTextSplitter(
|
|||
|
|
headers_to_split_on=[
|
|||
|
|
("#", "h1"),
|
|||
|
|
("##", "h2"),
|
|||
|
|
("###", "h3"),
|
|||
|
|
],
|
|||
|
|
strip_headers=False
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 备用的字符切分器
|
|||
|
|
self.char_splitter = RecursiveCharacterTextSplitter(
|
|||
|
|
chunk_size=800,
|
|||
|
|
chunk_overlap=100,
|
|||
|
|
separators=["\n\n", "\n", "。", "！", "？", " ", ""]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def clean_markdown(self, text: str) -> str:
|
|||
|
|
"""清洗 Markdown 中的网页噪音"""
|
|||
|
|
if not text: return ""
|
|||
|
|
|
|||
|
|
# 去除 'Skip to main content'
|
|||
|
|
text = re.sub(r'\[Skip to main content\].*?\n', '', text, flags=re.IGNORECASE)
|
|||
|
|
# 去除页脚导航 (Previous / Next)
|
|||
|
|
text = re.sub(r'\[Previous\].*?\[Next\].*', '', text, flags=re.DOTALL | re.IGNORECASE)
|
|||
|
|
|
|||
|
|
return text.strip()
|
|||
|
|
|
|||
|
|
def split_markdown(self, text: str):
|
|||
|
|
"""执行切分策略：先按标题切，过长则按字符切"""
|
|||
|
|
md_chunks = self.md_splitter.split_text(text)
|
|||
|
|
final_chunks = []
|
|||
|
|
|
|||
|
|
for chunk in md_chunks:
|
|||
|
|
# chunk.page_content 是文本
|
|||
|
|
# chunk.metadata 是标题层级
|
|||
|
|
|
|||
|
|
if len(chunk.page_content) > 1000:
|
|||
|
|
sub_texts = self.char_splitter.split_text(chunk.page_content)
|
|||
|
|
for sub in sub_texts:
|
|||
|
|
final_chunks.append({
|
|||
|
|
"content": sub,
|
|||
|
|
"metadata": chunk.metadata
|
|||
|
|
})
|
|||
|
|
else:
|
|||
|
|
final_chunks.append({
|
|||
|
|
"content": chunk.page_content,
|
|||
|
|
"metadata": chunk.metadata
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
return final_chunks
|
|||
|
|
|
|||
|
|
# 单例工具
|
|||
|
|
text_processor = TextProcessor()
|