zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复上传文档数据集时出现重复问题 (#465)

Browse Source
This commit is contained in:
shaohuzhang1 2024-05-15 17:22:54 +08:00 committed by GitHub
parent cad10bb6a9
commit e74ba87fc6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 18 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/dataset/serializers/dataset_serializers.py
View File

@ -316,7 +316,8 @@ class DataSetSerializers(serializers.ModelSerializer):
problem_model_list.append(problem) problem_model_list.append(problem)
for problem_paragraph_mapping in document_paragraph_dict_model.get('problem_paragraph_mapping_list'): for problem_paragraph_mapping in document_paragraph_dict_model.get('problem_paragraph_mapping_list'):
problem_paragraph_mapping_list.append(problem_paragraph_mapping) problem_paragraph_mapping_list.append(problem_paragraph_mapping)
problem_model_list, problem_paragraph_mapping_list = DocumentSerializers.Create.reset_problem_model(
problem_model_list, problem_paragraph_mapping_list)
# 插入知识库 # 插入知识库
dataset.save() dataset.save()
# 插入文档 # 插入文档

16
apps/dataset/serializers/document_serializers.py
View File

@ -549,10 +549,26 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
problem_paragraph_mapping_list.append(problem_paragraph_mapping) problem_paragraph_mapping_list.append(problem_paragraph_mapping)
paragraph_model_list.append(paragraph) paragraph_model_list.append(paragraph)
problem_model_list, problem_paragraph_mapping_list = DocumentSerializers.Create.reset_problem_model(
problem_model_list, problem_paragraph_mapping_list)
return {'document': document_model, 'paragraph_model_list': paragraph_model_list, return {'document': document_model, 'paragraph_model_list': paragraph_model_list,
'problem_model_list': problem_model_list, 'problem_model_list': problem_model_list,
'problem_paragraph_mapping_list': problem_paragraph_mapping_list} 'problem_paragraph_mapping_list': problem_paragraph_mapping_list}
@staticmethod
def reset_problem_model(problem_model_list, problem_paragraph_mapping_list):
new_problem_model_list = [x for i, x in enumerate(problem_model_list) if
len([item for item in problem_model_list[:i] if item.content == x.content]) <= 0]
for new_problem_model in new_problem_model_list:
old_model_list = [problem.id for problem in problem_model_list if
problem.content == new_problem_model.content]
for problem_paragraph_mapping in problem_paragraph_mapping_list:
if old_model_list.__contains__(problem_paragraph_mapping.problem_id):
problem_paragraph_mapping.problem_id = new_problem_model.id
return new_problem_model_list, problem_paragraph_mapping_list
@staticmethod @staticmethod
def get_document_paragraph_model(dataset_id, instance: Dict): def get_document_paragraph_model(dataset_id, instance: Dict):
document_model = Document( document_model = Document(