Merge branch 'pr@main@model_embedding' of github.com:1Panel-dev/MaxKB into pr@main@model_embedding

ui/src/api/type/dataset.ts

@@ -3,6 +3,7 @@ interface datasetData {
   desc: String
   documents?: Array<any>
   type?: String
+  embedding_mode_id?: String
 }
 
 export type { datasetData }

ui/src/router/modules/dataset.ts

@@ -13,9 +13,9 @@ const datasetRouter = {
     },
     {
       path: '/dataset/:type', // create 或者 upload
-      name: 'CreateDataset',
+      name: 'UploadDocumentDataset',
       meta: { activeMenu: '/dataset' },
-      component: () => import('@/views/dataset/CreateDataset.vue'),
+      component: () => import('@/views/dataset/UploadDocumentDataset.vue'),
       hidden: true
     },
     {

ui/src/views/application/component/CreateApplicationDialog.vue

@@ -63,10 +63,10 @@
     </el-form>
     <template #footer>
       <span class="dialog-footer">
-        <el-button @click.prevent="dialogVisible = false">
+        <el-button @click.prevent="dialogVisible = false" :loading="loading">
           {{ $t('views.application.applicationForm.buttons.cancel') }}
         </el-button>
-        <el-button type="primary" @click="submitValid(applicationFormRef)">
+        <el-button type="primary" @click="submitValid(applicationFormRef)" :loading="loading">
           {{ $t('views.application.applicationForm.buttons.create') }}
         </el-button>
       </span>

ui/src/views/dataset/DatasetSetting.vue

@@ -3,6 +3,7 @@
     <div class="dataset-setting main-calc-height">
       <el-scrollbar>
         <div class="p-24" v-loading="loading">
+          <h4 class="title-decoration-1 mb-16">基本信息</h4>
           <BaseForm ref="BaseFormRef" :data="detail" />
 
           <el-form

ui/src/views/dataset/UploadDocumentDataset.vue

@@ -1,15 +1,18 @@
 <template>
-  <LayoutContainer :header="isCreate ? '创建知识库' : '上传文档'" class="create-dataset">
+  <LayoutContainer header="上传文档" class="create-dataset">
     <template #backButton>
       <back-button @click="back"></back-button>
     </template>
     <div class="create-dataset__main flex" v-loading="loading">
       <div class="create-dataset__component main-calc-height">
         <template v-if="active === 0">
-          <StepFirst ref="StepFirstRef" />
+          <div class="upload-document p-24">
+            <!-- 上传文档 -->
+            <UploadComponent ref="UploadComponentRef" />
+          </div>
         </template>
         <template v-else-if="active === 1">
-          <StepSecond ref="StepSecondRef" />
+          <SetRules ref="SetRulesRef" />
         </template>
         <template v-else-if="active === 2">
           <ResultSuccess :data="successInfo" />
@@ -19,12 +22,7 @@
     <div class="create-dataset__footer text-right border-t" v-if="active !== 2">
       <el-button @click="router.go(-1)" :disabled="loading">取消</el-button>
       <el-button @click="prev" v-if="active === 1" :disabled="loading">上一步</el-button>
-      <el-button
+      <el-button @click="next" type="primary" v-if="active === 0" :disabled="loading">
-        @click="next"
-        type="primary"
-        v-if="active === 0"
-        :disabled="loading || StepFirstRef?.loading"
-      >
         创建并导入
       </el-button>
       <el-button @click="submit" type="primary" v-if="active === 1" :disabled="loading">
@@ -36,9 +34,9 @@
 <script setup lang="ts">
 import { ref, computed, onUnmounted } from 'vue'
 import { useRouter, useRoute } from 'vue-router'
-import StepFirst from './step/StepFirst.vue'
+import SetRules from './component/SetRules.vue'
-import StepSecond from './step/StepSecond.vue'
+import ResultSuccess from './component/ResultSuccess.vue'
-import ResultSuccess from './step/ResultSuccess.vue'
+import UploadComponent from './component/UploadComponent.vue'
 import datasetApi from '@/api/dataset'
 import documentApi from '@/api/document'
 import type { datasetData } from '@/api/type/dataset'
@@ -46,33 +44,17 @@ import { MsgConfirm, MsgSuccess } from '@/utils/message'
 
 import useStore from '@/stores'
 const { dataset, document } = useStore()
-const baseInfo = computed(() => dataset.baseInfo)
-const webInfo = computed(() => dataset.webInfo)
 const documentsFiles = computed(() => dataset.documentsFiles)
 const documentsType = computed(() => dataset.documentsType)
 
 const router = useRouter()
 const route = useRoute()
 const {
-  params: { type },
   query: { id } // id为datasetID，有id的是上传文档
 } = route
-const isCreate = type === 'create'
-// const steps = [
-//   {
-//     ref: 'StepFirstRef',
-//     name: '上传文档',
-//     component: StepFirst
-//   },
-//   {
-//     ref: 'StepSecondRef',
-//     name: '设置分段规则',
-//     component: StepSecond
-//   }
-// ]
 
-const StepFirstRef = ref()
+const SetRulesRef = ref()
-const StepSecondRef = ref()
+const UploadComponentRef = ref()
 
 const loading = ref(false)
 const disabled = ref(false)
@@ -81,7 +63,7 @@ const successInfo = ref<any>(null)
 
 async function next() {
   disabled.value = true
-  if (await StepFirstRef.value?.onSubmit()) {
+  if (await UploadComponentRef.value.validate()) {
     if (documentsType.value === 'QA') {
       let fd = new FormData()
       documentsFiles.value.forEach((item: any) => {
@@ -118,16 +100,14 @@ const prev = () => {
 }
 
 function clearStore() {
-  dataset.saveBaseInfo(null)
-  dataset.saveWebInfo(null)
   dataset.saveDocumentsFile([])
   dataset.saveDocumentsType('')
 }
 function submit() {
   loading.value = true
   const documents = [] as any
-  StepSecondRef.value?.paragraphList.map((item: any) => {
+  SetRulesRef.value?.paragraphList.map((item: any) => {
-    if (!StepSecondRef.value?.checkedConnect) {
+    if (!SetRulesRef.value?.checkedConnect) {
       item.content.map((v: any) => {
         delete v['problem_list']
       })
@@ -159,7 +139,7 @@ function submit() {
   }
 }
 function back() {
-  if (baseInfo.value || webInfo.value || documentsFiles.value?.length > 0) {
+  if (documentsFiles.value?.length > 0) {
     MsgConfirm(`提示`, `当前的更改尚未保存，确认退出吗?`, {
       confirmButtonText: '确认',
       type: 'warning'
@@ -206,5 +186,10 @@ onUnmounted(() => {
     width: 100%;
     box-sizing: border-box;
   }
+  .upload-document {
+    width: 70%;
+    margin: 0 auto;
+    margin-bottom: 20px;
+  }
 }
 </style>

ui/src/views/dataset/component/BaseForm.vue

@@ -1,5 +1,4 @@
 <template>
-  <h4 class="title-decoration-1 mb-16">基本信息</h4>
   <el-form
     ref="FormRef"
     :model="form"
@@ -27,14 +26,26 @@
         @blur="form.desc = form.desc.trim()"
       />
     </el-form-item>
+    <el-form-item label="Embedding模型" prop="embedding_mode_id">
+      <el-select
+        v-model="form.embedding_mode_id"
+        class="w-full m-2"
+        placeholder="请选择Embedding模型"
+      >
+        <el-option
+          v-for="item in modelOptions"
+          :key="item.id"
+          :label="item.name"
+          :value="item.id"
+        ></el-option>
+      </el-select>
+    </el-form-item>
   </el-form>
 </template>
 <script setup lang="ts">
 import { ref, reactive, onMounted, onUnmounted, computed, watch } from 'vue'
-import { useRoute } from 'vue-router'
 import useStore from '@/stores'
 import type { datasetData } from '@/api/type/dataset'
-import { isAllPropertiesEmpty } from '@/utils/utils'
 
 const props = defineProps({
   data: {
@@ -42,47 +53,33 @@ const props = defineProps({
     default: () => {}
   }
 })
-const route = useRoute()
+const { model } = useStore()
-const {
-  params: { type }
-} = route
-const isCreate = type === 'create'
-const { dataset } = useStore()
-const baseInfo = computed(() => dataset.baseInfo)
 const form = ref<datasetData>({
   name: '',
-  desc: ''
+  desc: '',
+  embedding_mode_id: ''
 })
 
 const rules = reactive({
   name: [{ required: true, message: '请输入知识库名称', trigger: 'blur' }],
-  desc: [{ required: true, message: '请输入知识库描述', trigger: 'blur' }]
+  desc: [{ required: true, message: '请输入知识库描述', trigger: 'blur' }],
+  embedding_mode_id: [{ required: true, message: '请输入Embedding模型', trigger: 'change' }]
 })
 const FormRef = ref()
+const modelOptions = ref([])
 
 watch(
   () => props.data,
   (value) => {
     if (value && JSON.stringify(value) !== '{}') {
       form.value.name = value.name
-      form.value.desc = value.desc
+      form.value.embedding_mode_id = value.embedding_mode_id
     }
   },
   {
     immediate: true
   }
 )
-
-watch(form.value, (value) => {
-  if (isAllPropertiesEmpty(value)) {
-    dataset.saveBaseInfo(null)
-  } else {
-    if (isCreate) {
-      dataset.saveBaseInfo(value)
-    }
-  }
-})
-
 /*
   表单校验
 */
@@ -93,16 +90,22 @@ function validate() {
   })
 }
 
+function getModel() {
+  model.asyncGetModel({ model_type: 'EMBEDDING' }).then((res: any) => {
+    modelOptions.value = res?.data
+  })
+}
+
 onMounted(() => {
-  if (baseInfo.value) {
+  getModel()
-    form.value = baseInfo.value
-  }
 })
 onUnmounted(() => {
   form.value = {
     name: '',
-    desc: ''
+    desc: '',
+    embedding_mode_id: ''
   }
+  FormRef.value?.clearValidate()
 })
 
 defineExpose({

ui/src/views/dataset/component/CreateDatasetDialog.vue

@@ -0,0 +1,176 @@
+<template>
+  <el-dialog title="创建知识库" v-model="dialogVisible" width="650" append-to-body>
+    <!-- 基本信息 -->
+    <BaseForm ref="BaseFormRef" v-if="dialogVisible" />
+    <el-form
+      ref="DatasetFormRef"
+      :rules="rules"
+      :model="datasetForm"
+      label-position="top"
+      require-asterisk-position="right"
+    >
+      <el-form-item label="知识库类型" required>
+        <el-radio-group v-model="datasetForm.type" class="card__radio" @change="radioChange">
+          <el-row :gutter="20">
+            <el-col :span="12">
+              <el-card
+                shadow="never"
+                class="mb-16"
+                :class="datasetForm.type === '0' ? 'active' : ''"
+              >
+                <el-radio value="0" size="large">
+                  <div class="flex align-center">
+                    <AppAvatar class="mr-8 avatar-blue" shape="square" :size="32">
+                      <img src="@/assets/icon_document.svg" style="width: 58%" alt="" />
+                    </AppAvatar>
+                    <div>
+                      <p class="mb-4">通用型</p>
+                      <el-text type="info">可以通过上传文件或手动录入方式构建知识库</el-text>
+                    </div>
+                  </div>
+                </el-radio>
+              </el-card>
+            </el-col>
+            <el-col :span="12">
+              <el-card
+                shadow="never"
+                class="mb-16"
+                :class="datasetForm.type === '1' ? 'active' : ''"
+              >
+                <el-radio value="1" size="large">
+                  <div class="flex align-center">
+                    <AppAvatar class="mr-8 avatar-purple" shape="square" :size="32">
+                      <img src="@/assets/icon_web.svg" style="width: 58%" alt="" />
+                    </AppAvatar>
+                    <div>
+                      <p class="mb-4">Web 站点</p>
+                      <el-text type="info">通过网站链接同步方式构建知识库 </el-text>
+                    </div>
+                  </div>
+                </el-radio>
+              </el-card>
+            </el-col>
+          </el-row>
+        </el-radio-group>
+      </el-form-item>
+      <el-form-item label="Web 根地址" prop="source_url" v-if="datasetForm.type === '1'">
+        <el-input
+          v-model="datasetForm.source_url"
+          placeholder="请输入 Web 根地址"
+          @blur="datasetForm.source_url = datasetForm.source_url.trim()"
+        />
+      </el-form-item>
+      <el-form-item label="选择器" v-if="datasetForm.type === '1'">
+        <el-input
+          v-model="datasetForm.selector"
+          placeholder="默认为 body，可输入 .classname/#idname/tagname"
+          @blur="datasetForm.selector = datasetForm.selector.trim()"
+        />
+      </el-form-item>
+    </el-form>
+    <template #footer>
+      <span class="dialog-footer">
+        <el-button @click.prevent="dialogVisible = false" :loading="loading">
+          {{ $t('views.application.applicationForm.buttons.cancel') }}
+        </el-button>
+        <el-button type="primary" @click="submitValid" :loading="loading">
+          {{ $t('views.application.applicationForm.buttons.create') }}
+        </el-button>
+      </span>
+    </template>
+  </el-dialog>
+</template>
+<script setup lang="ts">
+import { ref, watch, reactive } from 'vue'
+import { useRouter, useRoute } from 'vue-router'
+import BaseForm from './BaseForm.vue'
+import datasetApi from '@/api/dataset'
+import { MsgSuccess, MsgAlert } from '@/utils/message'
+import useStore from '@/stores'
+import { ValidType, ValidCount } from '@/enums/common'
+
+const { common, user } = useStore()
+const router = useRouter()
+const BaseFormRef = ref()
+const DatasetFormRef = ref()
+
+const loading = ref(false)
+const dialogVisible = ref<boolean>(false)
+
+const datasetForm = ref<any>({
+  type: '0',
+  source_url: '',
+  selector: ''
+})
+
+const rules = reactive({
+  source_url: [{ required: true, message: '请输入 Web 根地址', trigger: 'blur' }]
+})
+
+watch(dialogVisible, (bool) => {
+  if (!bool) {
+    datasetForm.value = {
+      type: '0',
+      source_url: '',
+      selector: ''
+    }
+    DatasetFormRef.value?.clearValidate()
+  }
+})
+
+const open = () => {
+  dialogVisible.value = true
+}
+
+const submitValid = () => {
+  if (user.isEnterprise()) {
+    submitHandle()
+  } else {
+    common.asyncGetValid(ValidType.Dataset, ValidCount.Dataset, loading).then(async (res: any) => {
+      if (res?.data) {
+        submitHandle()
+      } else {
+        MsgAlert(
+          '提示',
+          '社区版最多支持 50 个知识库，如需拥有更多知识库，请联系我们（https://fit2cloud.com/）。'
+        )
+      }
+    })
+  }
+}
+const submitHandle = async () => {
+  if (await BaseFormRef.value?.validate()) {
+    await DatasetFormRef.value.validate((valid: any) => {
+      if (valid) {
+        if (datasetForm.value.type === '0') {
+          const obj = {
+            ...BaseFormRef.value.form,
+            type: datasetForm.value.type
+          }
+          datasetApi.postDataset(obj, loading).then((res) => {
+            MsgSuccess('创建成功')
+            router.push({ path: `/dataset/${res.data.id}/document` })
+          })
+        } else {
+          const obj = { ...BaseFormRef.value.form, ...datasetForm.value }
+          datasetApi.postWebDataset(obj, loading).then((res) => {
+            MsgSuccess('创建成功')
+            router.push({ path: `/dataset/${res.data.id}/document` })
+          })
+        }
+      } else {
+        return false
+      }
+    })
+  } else {
+    return false
+  }
+}
+function radioChange() {
+  datasetForm.value.source_url = ''
+  datasetForm.value.selector = ''
+}
+
+defineExpose({ open })
+</script>
+<style lang="scss" scope></style>

ui/src/views/dataset/component/SetRules.vue

@@ -65,7 +65,7 @@
                           show-input
                           :show-input-controls="false"
                           :min="50"
-                          :max="4096"
+                          :max="100000"
                         />
                       </div>
                       <div class="form-item mb-16">

ui/src/views/dataset/index.vue

@@ -22,7 +22,7 @@
       >
         <el-row :gutter="15">
           <el-col :xs="24" :sm="12" :md="8" :lg="6" :xl="4" class="mb-16">
-            <CardAdd title="创建知识库" @click="router.push({ path: '/dataset/create' })" />
+            <CardAdd title="创建知识库" @click="openCreateDialog" />
           </el-col>
           <template v-for="(item, index) in datasetList" :key="index">
             <el-col :xs="24" :sm="12" :md="8" :lg="6" :xl="4" class="mb-16">
@@ -107,17 +107,20 @@
       </InfiniteScroll>
     </div>
     <SyncWebDialog ref="SyncWebDialogRef" @refresh="refresh" />
+    <CreateDatasetDialog ref="CreateDatasetDialogRef"/>
   </div>
 </template>
 <script setup lang="ts">
 import { ref, onMounted, reactive, computed } from 'vue'
 import SyncWebDialog from '@/views/dataset/component/SyncWebDialog.vue'
+import CreateDatasetDialog from './component/CreateDatasetDialog.vue'
 import datasetApi from '@/api/dataset'
 import { MsgSuccess, MsgConfirm } from '@/utils/message'
 import { useRouter } from 'vue-router'
 import { numberFormat } from '@/utils/utils'
 const router = useRouter()
 
+const CreateDatasetDialogRef = ref()
 const SyncWebDialogRef = ref()
 const loading = ref(false)
 const datasetList = ref<any[]>([])
@@ -129,6 +132,10 @@ const paginationConfig = reactive({
 
 const searchValue = ref('')
 
+function openCreateDialog() {
+  CreateDatasetDialogRef.value.open()
+}
+
 function refresh() {
   MsgSuccess('同步任务发送成功')
 }

ui/src/views/dataset/step/StepFirst.vue

@@ -1,183 +0,0 @@
-<template>
-  <el-scrollbar>
-    <div class="upload-document p-24">
-      <!-- 基本信息 -->
-      <BaseForm ref="BaseFormRef" v-if="isCreate" />
-      <el-form
-        v-if="isCreate"
-        ref="webFormRef"
-        :rules="rules"
-        :model="form"
-        label-position="top"
-        require-asterisk-position="right"
-      >
-        <el-form-item label="知识库类型" required>
-          <el-radio-group v-model="form.type" class="card__radio" @change="radioChange">
-            <el-row :gutter="20">
-              <el-col :span="12">
-                <el-card shadow="never" class="mb-16" :class="form.type === '0' ? 'active' : ''">
-                  <el-radio value="0" size="large">
-                    <div class="flex align-center">
-                      <AppAvatar class="mr-8" shape="square" :size="32">
-                        <img src="@/assets/icon_document.svg" style="width: 58%" alt="" />
-                      </AppAvatar>
-                      <div>
-                        <p class="mb-4">通用型</p>
-                        <el-text type="info">可以通过上传文件或手动录入方式构建知识库</el-text>
-                      </div>
-                    </div>
-                  </el-radio>
-                </el-card>
-              </el-col>
-              <el-col :span="12">
-                <el-card shadow="never" class="mb-16" :class="form.type === '1' ? 'active' : ''">
-                  <el-radio value="1" size="large">
-                    <div class="flex align-center">
-                      <AppAvatar class="mr-8 avatar-purple" shape="square" :size="32">
-                        <img src="@/assets/icon_web.svg" style="width: 58%" alt="" />
-                      </AppAvatar>
-                      <div>
-                        <p class="mb-4">Web 站点</p>
-                        <el-text type="info">通过网站链接同步方式构建知识库 </el-text>
-                      </div>
-                    </div>
-                  </el-radio>
-                </el-card>
-              </el-col>
-            </el-row>
-          </el-radio-group>
-        </el-form-item>
-        <el-form-item label="Web 根地址" prop="source_url" v-if="form.type === '1'">
-          <el-input
-            v-model="form.source_url"
-            placeholder="请输入 Web 根地址"
-            @blur="form.source_url = form.source_url.trim()"
-          />
-        </el-form-item>
-        <el-form-item label="选择器" v-if="form.type === '1'">
-          <el-input
-            v-model="form.selector"
-            placeholder="默认为 body，可输入 .classname/#idname/tagname"
-            @blur="form.selector = form.selector.trim()"
-          />
-        </el-form-item>
-      </el-form>
-
-      <!-- 上传文档 -->
-      <UploadComponent ref="UploadComponentRef" v-if="form.type === '0'" />
-    </div>
-  </el-scrollbar>
-</template>
-<script setup lang="ts">
-import { ref, onMounted, reactive, watch } from 'vue'
-import { useRouter, useRoute } from 'vue-router'
-import BaseForm from '@/views/dataset/component/BaseForm.vue'
-import UploadComponent from '@/views/dataset/component/UploadComponent.vue'
-import { isAllPropertiesEmpty } from '@/utils/utils'
-import datasetApi from '@/api/dataset'
-import { MsgError, MsgSuccess } from '@/utils/message'
-import useStore from '@/stores'
-const { dataset } = useStore()
-
-const route = useRoute()
-const router = useRouter()
-const {
-  params: { type }
-} = route
-const isCreate = type === 'create'
-const BaseFormRef = ref()
-const UploadComponentRef = ref()
-const webFormRef = ref()
-const loading = ref(false)
-
-const form = ref<any>({
-  type: '0',
-  source_url: '',
-  selector: ''
-})
-
-const rules = reactive({
-  source_url: [{ required: true, message: '请输入 Web 根地址', trigger: 'blur' }]
-})
-
-watch(form.value, (value) => {
-  if (isAllPropertiesEmpty(value)) {
-    dataset.saveWebInfo(null)
-  } else {
-    dataset.saveWebInfo(value)
-  }
-})
-
-function radioChange() {
-  dataset.saveDocumentsFile([])
-  dataset.saveDocumentsType('')
-  form.value.source_url = ''
-  form.value.selector = ''
-}
-
-const onSubmit = async () => {
-  if (isCreate) {
-    if (form.value.type === '0') {
-      if ((await BaseFormRef.value?.validate()) && (await UploadComponentRef.value.validate())) {
-        if (UploadComponentRef.value.form.fileList.length > 50) {
-          MsgError('每次最多上传50个文件！')
-          return false
-        } else {
-          /*
-        stores保存数据
-      */
-          dataset.saveBaseInfo(BaseFormRef.value.form)
-          dataset.saveDocumentsType(UploadComponentRef.value.form.fileType)
-          dataset.saveDocumentsFile(UploadComponentRef.value.form.fileList)
-          return true
-        }
-      } else {
-        return false
-      }
-    } else {
-      if (await BaseFormRef.value?.validate()) {
-        await webFormRef.value.validate((valid: any) => {
-          if (valid) {
-            const obj = { ...BaseFormRef.value.form, ...form.value }
-            datasetApi.postWebDataset(obj, loading).then((res) => {
-              MsgSuccess('提交成功')
-              dataset.saveBaseInfo(null)
-              dataset.saveWebInfo(null)
-              router.push({ path: `/dataset/${res.data.id}/document` })
-            })
-          } else {
-            return false
-          }
-        })
-      } else {
-        return false
-      }
-    }
-  } else {
-    if (await UploadComponentRef.value.validate()) {
-      /*
-        stores保存数据
-      */
-      dataset.saveDocumentsType(UploadComponentRef.value.form.fileType)
-      dataset.saveDocumentsFile(UploadComponentRef.value.form.fileList)
-      return true
-    } else {
-      return false
-    }
-  }
-}
-
-onMounted(() => {})
-
-defineExpose({
-  onSubmit,
-  loading
-})
-</script>
-<style scoped lang="scss">
-.upload-document {
-  width: 70%;
-  margin: 0 auto;
-  margin-bottom: 20px;
-}
-</style>

ui/src/views/paragraph/component/ParagraphForm.vue

@@ -23,7 +23,7 @@
         v-if="isEdit"
         v-model="form.content"
         placeholder="请输入分段内容"
-        :maxLength="4096"
+        :maxLength="100000"
         :preview="false"
         :toolbars="toolbars"
         style="height: 300px"
@@ -31,7 +31,7 @@
         :footers="footers"
       >
         <template #defFooters>
-          <span style="margin-left: -6px">/ 4096</span>
+          <span style="margin-left: -6px">/ 100000</span>
         </template>
       </MdEditor>
       <MdPreview