37 lines
1.1 KiB
Python
37 lines
1.1 KiB
Python
"""Transcript chunker — split raw text into retrieval-ready chunks."""
|
||
|
||
|
||
def chunk_transcript(
|
||
text: str, *, max_chars: int = 800, overlap_chars: int = 100
|
||
) -> list[str]:
|
||
"""
|
||
Split transcript text into overlapping chunks.
|
||
Uses character count as proxy for tokens (~4 chars/token for Chinese).
|
||
"""
|
||
if not text or not text.strip():
|
||
return []
|
||
text = text.strip()
|
||
if len(text) <= max_chars:
|
||
return [text] if text else []
|
||
|
||
chunks: list[str] = []
|
||
start = 0
|
||
step = max_chars - overlap_chars
|
||
|
||
while start < len(text):
|
||
end = start + max_chars
|
||
chunk = text[start:end]
|
||
# 尽量在句末切分
|
||
if end < len(text):
|
||
for sep in ["。", "！", "？", "\n", "；", ".", "!", "?"]:
|
||
last_sep = chunk.rfind(sep)
|
||
if last_sep > max_chars // 2:
|
||
chunk = chunk[: last_sep + 1]
|
||
end = start + len(chunk)
|
||
break
|
||
if chunk.strip():
|
||
chunks.append(chunk.strip())
|
||
start += len(chunk) if chunk else step
|
||
|
||
return chunks
|