zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 智能分段文本分段丢失数据

Browse Source
This commit is contained in:
zhangshaohu 2024-03-23 19:46:20 +08:00
parent 0c9a7c15b6
commit 53d45e069d
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/common/util/split_model.py
View File

@ -358,7 +358,7 @@ default_split_pattern = {
'md': [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#)## (?!#).*'), re.compile("(?<!#)### (?!#).*"), 'md': [re.compile('(?<=^)# .*|(?<=\\n)# .*'), re.compile('(?<!#)## (?!#).*'), re.compile("(?<!#)### (?!#).*"),
re.compile("(?<!#)#### (?!#).*"), re.compile("(?<!#)##### (?!#).*"), re.compile("(?<!#)#### (?!#).*"), re.compile("(?<!#)##### (?!#).*"),
re.compile("(?<!#)###### (?!#).*")], re.compile("(?<!#)###### (?!#).*")],
'default': [re.compile("(?<!\n)\n\n.+")] 'default': [re.compile("(?<!\n)\n\n+")]
} }