feat: 修改标注时可以选择默认知识库

--story=1016833 --user=王孝刚 对话日志-修改标注时可以选择默认知识库 #229 https://www.tapd.cn/57709429/s/1608846

apps/application/serializers/chat_serializers.py

@@ -40,7 +40,7 @@ from common.util.lock import try_lock, un_lock
 from dataset.models import Document, Problem, Paragraph, ProblemParagraphMapping
 from dataset.serializers.common_serializers import get_embedding_model_id_by_dataset_id
 from dataset.serializers.paragraph_serializers import ParagraphSerializers
-from embedding.task import embedding_by_paragraph
+from embedding.task import embedding_by_paragraph, embedding_by_paragraph_list
 from setting.models import Model
 from setting.models_provider import get_model_credential
 from smartdoc.conf import PROJECT_DIR
@@ -658,3 +658,67 @@ class ChatRecordSerializer(serializers.Serializer):
                     data={"dataset_id": dataset_id, 'document_id': document_id, "paragraph_id": paragraph_id})
                 o.is_valid(raise_exception=True)
                 return o.delete()
+
+    class PostImprove(serializers.Serializer):
+        dataset_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid("知识库id"))
+        document_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid("文档id"))
+        chat_ids = serializers.ListSerializer(child=serializers.UUIDField(), required=True,
+                                              error_messages=ErrMessage.list("对话id"))
+
+        def is_valid(self, *, raise_exception=False):
+            super().is_valid(raise_exception=True)
+            if not Document.objects.filter(id=self.data['document_id'], dataset_id=self.data['dataset_id']).exists():
+                raise AppApiException(500, "文档id不正确")
+
+        @staticmethod
+        def post_embedding_paragraph(paragraph_ids, dataset_id):
+            model_id = get_embedding_model_id_by_dataset_id(dataset_id)
+            embedding_by_paragraph_list(paragraph_ids, model_id)
+
+        @post(post_function=post_embedding_paragraph)
+        @transaction.atomic
+        def post_improve(self, instance: Dict):
+            ChatRecordSerializer.PostImprove(data=instance).is_valid(raise_exception=True)
+
+            chat_ids = instance['chat_ids']
+            document_id = instance['document_id']
+            dataset_id = instance['dataset_id']
+
+            # 获取所有聊天记录
+            chat_record_list = list(ChatRecord.objects.filter(chat_id__in=chat_ids))
+            if len(chat_record_list) < len(chat_ids):
+                raise AppApiException(500, "存在不存在的对话记录")
+
+            # 批量创建段落和问题映射
+            paragraphs = []
+            paragraph_ids = []
+            problem_paragraph_mappings = []
+            for chat_record in chat_record_list:
+                paragraph = Paragraph(
+                    id=uuid.uuid1(),
+                    document_id=document_id,
+                    content=chat_record.answer_text,
+                    dataset_id=dataset_id,
+                    title=chat_record.problem_text
+                )
+                problem, _ = Problem.objects.get_or_create(content=chat_record.problem_text, dataset_id=dataset_id)
+                problem_paragraph_mapping = ProblemParagraphMapping(
+                    id=uuid.uuid1(),
+                    dataset_id=dataset_id,
+                    document_id=document_id,
+                    problem_id=problem.id,
+                    paragraph_id=paragraph.id
+                )
+                paragraphs.append(paragraph)
+                paragraph_ids.append(paragraph.id)
+                problem_paragraph_mappings.append(problem_paragraph_mapping)
+                chat_record.improve_paragraph_id_list.append(paragraph.id)
+
+            # 批量保存段落和问题映射
+            Paragraph.objects.bulk_create(paragraphs)
+            ProblemParagraphMapping.objects.bulk_create(problem_paragraph_mappings)
+
+            # 批量保存聊天记录
+            ChatRecord.objects.bulk_update(chat_record_list, ['improve_paragraph_id_list'])
+
+            return paragraph_ids, dataset_id

apps/application/swagger_api/chat_api.py

@@ -267,6 +267,38 @@ class ImproveApi(ApiMixin):
             }
         )
 
+    @staticmethod
+    def get_request_body_api_post():
+        return openapi.Schema(
+            type=openapi.TYPE_OBJECT,
+            required=['dataset_id', 'document_id', 'chat_ids'],
+            properties={
+                'dataset_id': openapi.Schema(type=openapi.TYPE_STRING, title="知识库id",
+                                             description="知识库id"),
+                'document_id': openapi.Schema(type=openapi.TYPE_STRING, title="文档id",
+                                              description="文档id"),
+                'chat_ids': openapi.Schema(type=openapi.TYPE_ARRAY, title="会话id列表",
+                                           description="会话id列表",
+                                           items=openapi.Schema(type=openapi.TYPE_STRING))
+
+            }
+        )
+
+    @staticmethod
+    def get_request_params_api_post():
+        return [openapi.Parameter(name='application_id',
+                                  in_=openapi.IN_PATH,
+                                  type=openapi.TYPE_STRING,
+                                  required=True,
+                                  description='应用id'),
+                openapi.Parameter(name='dataset_id',
+                                  in_=openapi.IN_PATH,
+                                  type=openapi.TYPE_STRING,
+                                  required=True,
+                                  description='知识库id'),
+
+                ]
+
 
 class VoteApi(ApiMixin):
     @staticmethod

apps/application/urls.py

@@ -61,6 +61,10 @@ urlpatterns = [
         'application/<str:application_id>/chat/<chat_id>/chat_record/<str:chat_record_id>/dataset/<str:dataset_id>/document_id/<str:document_id>/improve',
         views.ChatView.ChatRecord.Improve.as_view(),
         name=''),
+    path(
+        'application/<str:application_id>/dataset/<str:dataset_id>/improve',
+        views.ChatView.ChatRecord.Improve.as_view(),
+        name=''),
     path('application/<str:application_id>/chat/<chat_id>/chat_record/<str:chat_record_id>/improve',
          views.ChatView.ChatRecord.ChatRecordImprove.as_view()),
     path('application/chat_message/<str:chat_id>', views.ChatView.Message.as_view(), name='application/message'),

apps/application/views/chat_views.py

@@ -129,7 +129,8 @@ class ChatView(APIView):
                                                'client_id': request.auth.client_id,
                                                'form_data': (request.data.get(
                                                    'form_data') if 'form_data' in request.data else {}),
-                                               'image_list': request.data.get('image_list') if 'image_list' in request.data else [],
+                                               'image_list': request.data.get(
+                                                   'image_list') if 'image_list' in request.data else [],
                                                'client_type': request.auth.client_type}).chat()
 
     @action(methods=['GET'], detail=False)
@@ -364,6 +365,28 @@ class ChatView(APIView):
                     data={'chat_id': chat_id, 'chat_record_id': chat_record_id,
                           'dataset_id': dataset_id, 'document_id': document_id}).improve(request.data))
 
+            @action(methods=['POST'], detail=False)
+            @swagger_auto_schema(operation_summary="添加至知识库",
+                                 operation_id="添加至知识库",
+                                 manual_parameters=ImproveApi.get_request_params_api_post(),
+                                 request_body=ImproveApi.get_request_body_api_post(),
+                                 tags=["应用/对话日志/添加至知识库"]
+                                 )
+            @has_permissions(
+                ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
+                               [lambda r, keywords: Permission(group=Group.APPLICATION, operate=Operate.USE,
+                                                               dynamic_tag=keywords.get('application_id'))],
+
+                               ), ViewPermission([RoleConstants.ADMIN, RoleConstants.USER],
+                                                 [lambda r, keywords: Permission(group=Group.DATASET,
+                                                                                 operate=Operate.MANAGE,
+                                                                                 dynamic_tag=keywords.get(
+                                                                                     'dataset_id'))],
+                                                 compare=CompareConstants.AND
+                                                 ), compare=CompareConstants.AND)
+            def post(self, request: Request, application_id: str, dataset_id: str):
+                return result.success(ChatRecordSerializer.PostImprove().post_improve(request.data))
+
             class Operate(APIView):
                 authentication_classes = [TokenAuth]
 
@@ -417,4 +440,3 @@ class ChatView(APIView):
                 file_url = FileSerializer(data={'file': file, 'meta': meta}).upload()
                 file_ids.append({'name': file.name, 'url': file_url, 'file_id': file_url.split('/')[-1]})
             return result.success(file_ids)
-

ui/src/api/log.ts

@@ -1,5 +1,5 @@
 import { Result } from '@/request/Result'
-import { get, del, put, exportExcel, exportExcelPost } from '@/request/index'
+import { get, del, put, exportExcel, exportExcelPost, post } from '@/request/index'
 import type { pageRequest } from '@/api/type/common'
 import { type Ref } from 'vue'
 
@@ -114,7 +114,22 @@ const putChatRecordLog: (
     loading
   )
 }
+/**
+ * 对话记录提交至知识库
+ * @param data
+ * @param loading
+ * @param application_id
+ * @param dataset_id
+ */
 
+const postChatRecordLog: (
+  application_id: string,
+  dataset_id: string,
+  data: any,
+  loading?: Ref<boolean>
+) => Promise<Result<any>> = (application_id, dataset_id, data, loading) => {
+  return post(`${prefix}/${application_id}/dataset/${dataset_id}/improve`, data, undefined, loading)
+}
 /**
  * 获取标注段落列表信息
  * @param 参数
@@ -215,5 +230,6 @@ export default {
   delMarkRecord,
   exportChatLog,
   getChatLogClient,
-  delChatClientLog
+  delChatClientLog,
+  postChatRecordLog
 }

ui/src/views/log/component/EditContentDialog.vue

@@ -85,6 +85,7 @@
           filterable
           placeholder="请选择文档"
           :loading="optionLoading"
+          @change="changeDocument"
         >
           <el-option
             v-for="item in documentList"
@@ -113,7 +114,7 @@ import logApi from '@/api/log'
 import imageApi from '@/api/image'
 import useStore from '@/stores'
 
-const { application, document } = useStore()
+const { application, document, user } = useStore()
 
 const route = useRoute()
 const {
@@ -216,10 +217,19 @@ const onUploadImg = async (files: any, callback: any) => {
 }
 
 function changeDataset(id: string) {
+  if (user.userInfo) {
+    localStorage.setItem(user.userInfo.id + 'chat_dataset_id', id)
+  }
   form.value.document_id = ''
   getDocument(id)
 }
 
+function changeDocument(id: string) {
+  if (user.userInfo) {
+    localStorage.setItem(user.userInfo.id + 'chat_document_id', id)
+  }
+}
+
 function getDocument(id: string) {
   document.asyncGetAllDocument(id, loading).then((res: any) => {
     documentList.value = res.data
@@ -229,11 +239,22 @@ function getDocument(id: string) {
 function getDataset() {
   application.asyncGetApplicationDataset(id, loading).then((res: any) => {
     datasetList.value = res.data
+    if (localStorage.getItem(user.userInfo?.id + 'chat_dataset_id')) {
+      form.value.dataset_id = localStorage.getItem(user.userInfo?.id + 'chat_dataset_id') as string
+      if (!datasetList.value.find((v) => v.id === form.value.dataset_id)) {
+        form.value.dataset_id = ''
+      } else {
+        getDocument(form.value.dataset_id)
+      }
+    }
   })
 }
 
 const open = (data: any) => {
   getDataset()
+  if (localStorage.getItem(user.userInfo?.id + 'chat_document_id')) {
+    form.value.document_id = localStorage.getItem(user.userInfo?.id + 'chat_document_id') as string
+  }
   form.value.chat_id = data.chat_id
   form.value.record_id = data.id
   form.value.problem_text = data.problem_text ? data.problem_text.substring(0, 256) : ''

ui/src/views/log/index.vue

@@ -30,8 +30,11 @@
           clearable
         />
         <div style="display: flex; align-items: center" class="float-right">
-          <el-button @click="dialogVisible = true" type="primary">清除策略</el-button>
+          <el-button @click="dialogVisible = true">清除策略</el-button>
           <el-button @click="exportLog">导出</el-button>
+          <el-button @click="openDocumentDialog" :disabled="multipleSelection.length === 0"
+            >添加至知识库</el-button
+          >
         </div>
       </div>
 
@@ -177,6 +180,86 @@
         </div>
       </template>
     </el-dialog>
+
+    <el-dialog
+      title="添加至知识库"
+      v-model="documentDialogVisible"
+      width="50%"
+      :close-on-click-modal="false"
+      :close-on-press-escape="false"
+    >
+      <el-form
+        ref="formRef"
+        :model="form"
+        label-position="top"
+        require-asterisk-position="right"
+        :rules="rules"
+        @submit.prevent
+      >
+        <el-form-item label="选择知识库" prop="dataset_id">
+          <el-select
+            v-model="form.dataset_id"
+            filterable
+            placeholder="请选择知识库"
+            :loading="optionLoading"
+            @change="changeDataset"
+          >
+            <el-option
+              v-for="item in datasetList"
+              :key="item.id"
+              :label="item.name"
+              :value="item.id"
+            >
+              <span class="flex align-center">
+                <AppAvatar
+                  v-if="!item.dataset_id && item.type === '1'"
+                  class="mr-12 avatar-purple"
+                  shape="square"
+                  :size="24"
+                >
+                  <img src="@/assets/icon_web.svg" style="width: 58%" alt="" />
+                </AppAvatar>
+                <AppAvatar
+                  v-else-if="!item.dataset_id && item.type === '0'"
+                  class="mr-12 avatar-blue"
+                  shape="square"
+                  :size="24"
+                >
+                  <img src="@/assets/icon_document.svg" style="width: 58%" alt="" />
+                </AppAvatar>
+                {{ item.name }}
+              </span>
+            </el-option>
+          </el-select>
+        </el-form-item>
+        <el-form-item label="保存至文档" prop="document_id">
+          <el-select
+            v-model="form.document_id"
+            filterable
+            placeholder="请选择文档"
+            :loading="optionLoading"
+            @change="changeDocument"
+          >
+            <el-option
+              v-for="item in documentList"
+              :key="item.id"
+              :label="item.name"
+              :value="item.id"
+            >
+              {{ item.name }}
+            </el-option>
+          </el-select>
+        </el-form-item>
+      </el-form>
+      <template #footer>
+        <span class="dialog-footer">
+          <el-button @click.prevent="dialogVisible = false"> 取消 </el-button>
+          <el-button type="primary" @click="submitForm(formRef)" :loading="documentLoading">
+            保存
+          </el-button>
+        </span>
+      </template>
+    </el-dialog>
   </LayoutContainer>
 </template>
 <script setup lang="ts">
@@ -190,12 +273,16 @@ import { beforeDay, datetimeFormat, nowDate } from '@/utils/time'
 import useStore from '@/stores'
 import type { Dict } from '@/api/type/common'
 import { t } from '@/locales'
+import type { FormInstance, FormRules } from 'element-plus'
 
-const { application, log } = useStore()
+const { application, log, document, user } = useStore()
 const route = useRoute()
 const {
   params: { id }
-} = route
+} = route as any
+
+const emit = defineEmits(['refresh'])
+const formRef = ref()
 
 const dayOptions = [
   {
@@ -230,12 +317,14 @@ const multipleSelection = ref<any[]>([])
 
 const ChatRecordRef = ref()
 const loading = ref(false)
+const documentLoading = ref(false)
 const paginationConfig = reactive({
   current_page: 1,
   page_size: 20,
   total: 0
 })
 const dialogVisible = ref(false)
+const documentDialogVisible = ref(false)
 const days = ref<number>(180)
 const tableData = ref<any[]>([])
 const tableIndexMap = computed<Dict<number>>(() => {
@@ -264,6 +353,18 @@ const filter = ref<any>({
   comparer: 'and'
 })
 
+const form = ref<any>({
+  dataset_id: '',
+  document_id: ''
+})
+
+const rules = reactive<FormRules>({
+  dataset_id: [{ required: true, message: '请选择知识库', trigger: 'change' }],
+  document_id: [{ required: true, message: '请选择文档', trigger: 'change' }]
+})
+
+const optionLoading = ref(false)
+const documentList = ref<any[]>([])
 function filterChange(val: string) {
   if (val === 'clear') {
     filter.value = cloneDeep(defaultFilter)
@@ -441,6 +542,72 @@ function saveCleanTime() {
     })
 }
 
+function changeDataset(id: string) {
+  if (user.userInfo) {
+    localStorage.setItem(user.userInfo.id + 'chat_dataset_id', id)
+  }
+  form.value.document_id = ''
+  getDocument(id)
+}
+
+function changeDocument(id: string) {
+  if (user.userInfo) {
+    localStorage.setItem(user.userInfo.id + 'chat_document_id', id)
+  }
+}
+
+const datasetList = ref<any[]>([])
+function getDataset() {
+  application.asyncGetApplicationDataset(id, documentLoading).then((res: any) => {
+    datasetList.value = res.data
+    if (localStorage.getItem(user.userInfo?.id + 'chat_dataset_id')) {
+      form.value.dataset_id = localStorage.getItem(user.userInfo?.id + 'chat_dataset_id') as string
+      if (!datasetList.value.find((v) => v.id === form.value.dataset_id)) {
+        form.value.dataset_id = ''
+      } else {
+        getDocument(form.value.dataset_id)
+      }
+    }
+  })
+}
+
+const submitForm = async (formEl: FormInstance | undefined) => {
+  if (!formEl) return
+  const arr: string[] = []
+  multipleSelection.value.map((v) => {
+    if (v) {
+      arr.push(v.id)
+    }
+  })
+  await formEl.validate((valid) => {
+    if (valid) {
+      const obj = {
+        document_id: form.value.document_id,
+        dataset_id: form.value.dataset_id,
+        chat_ids: arr
+      }
+      logApi.postChatRecordLog(id, form.value.dataset_id, obj, documentLoading).then((res: any) => {
+        multipleSelection.value = []
+        documentDialogVisible.value = false
+      })
+    }
+  })
+}
+
+function getDocument(id: string) {
+  document.asyncGetAllDocument(id, documentLoading).then((res: any) => {
+    documentList.value = res.data
+  })
+}
+
+function openDocumentDialog() {
+  getDataset()
+  if (localStorage.getItem(user.userInfo?.id + 'chat_document_id')) {
+    form.value.document_id = localStorage.getItem(user.userInfo?.id + 'chat_document_id') as string
+  }
+  formRef.value?.clearValidate()
+  documentDialogVisible.value = true
+}
 onMounted(() => {
   changeDayHandle(history_day.value)
   getDetail()