feat: 创建数据集向量化,以文档的粒度向量化

apps/common/event/listener_manage.py

@@ -96,23 +96,9 @@ class ListenerManagement:
         :param dataset_id: 知识库id
         :return: None
         """
-        status = Status.success
+        document_list = QuerySet(Document).filter(dataset_id=dataset_id)
-        try:
+        for document in document_list:
-            data_list = native_search(
+            ListenerManagement.embedding_by_document(document.id)
-                {'problem': QuerySet(get_dynamics_model({'problem.dataset_id': django.db.models.CharField()})).filter(
-                    **{'problem.dataset_id': dataset_id}),
-                    'paragraph': QuerySet(Paragraph).filter(dataset_id=dataset_id)},
-                select_string=get_file_content(
-                    os.path.join(PROJECT_DIR, "apps", "common", 'sql', 'list_embedding_text.sql')))
-            # 删除知识库相关向量数据
-            VectorStore.get_embedding_vector().delete_by_dataset_id(dataset_id)
-            # 批量向量化
-            VectorStore.get_embedding_vector().batch_save(data_list)
-        except Exception as e:
-            status = Status.error
-        #  修改文档 以及段落的状态
-        QuerySet(Document).filter(dataset_id=dataset_id).update(**{'status': status})
-        QuerySet(Paragraph).filter(dataset_id=dataset_id).update(**{'status': status})
 
     @staticmethod
     def delete_embedding_by_document(document_id):

ui/src/views/document/index.vue

@@ -132,7 +132,7 @@ const paginationConfig = reactive({
 const initInterval = () => {
   interval = setInterval(() => {
     if (documentData.value.length > 0 && documentData.value.every((item) => item.status === '0')) {
-      getList()
+      getList(true)
     }
   }, 6000)
 }
@@ -238,7 +238,7 @@ function cellMouseLeave() {
 
 function handleSizeChange() {
   paginationConfig.current_page = 1
-  getList(true)
+  getList()
 }
 
 function getList(bool?: boolean) {