feat: 对话日志增加定时清理

apps/application/migrations/0018_application_clean_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-10-16 14:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('application', '0017_application_tts_model_params_setting'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='application',
+            name='clean_time',
+            field=models.IntegerField(default=180, verbose_name='清理时间'),
+        ),
+    ]

apps/application/models/application.py

@@ -65,6 +65,7 @@ class Application(AppModelMixin):
     tts_model_enable = models.BooleanField(verbose_name="语音合成模型是否启用", default=False)
     stt_model_enable = models.BooleanField(verbose_name="语音识别模型是否启用", default=False)
     tts_type = models.CharField(verbose_name="语音播放类型", max_length=20, default="BROWSER")
+    clean_time = models.IntegerField(verbose_name="清理时间", default=180)
 
     @staticmethod
     def get_default_model_prompt():

apps/application/serializers/application_serializers.py

@@ -743,13 +743,30 @@ class ApplicationSerializer(serializers.Serializer):
                 application_setting = QuerySet(application_setting_model).filter(
                     application_id=application_access_token.application_id).first()
                 if application_setting is not None:
+                    custom_theme = getattr(application_setting, 'custom_theme', {})
+                    float_location = getattr(application_setting, 'float_location', {})
+                    if not custom_theme:
+                        application_setting.custom_theme = {
+                            'theme_color': '',
+                            'header_font_color': ''
+                        }
+                    if not float_location:
+                        application_setting.float_location = {
+                            'x': {'type': '', 'value': ''},
+                            'y': {'type': '', 'value': ''}
+                        }
                     application_setting_dict = {'show_source': application_access_token.show_source,
                                                 'show_history': application_setting.show_history,
                                                 'draggable': application_setting.draggable,
                                                 'show_guide': application_setting.show_guide,
                                                 'avatar': application_setting.avatar,
                                                 'float_icon': application_setting.float_icon,
-                                                'authentication': application_setting.authentication}
+                                                'authentication': application_setting.authentication,
+                                                'disclaimer': application_setting.disclaimer,
+                                                'disclaimer_value': application_setting.disclaimer_value,
+                                                'custom_theme': application_setting.custom_theme,
+                                                'user_avatar': application_setting.user_avatar,
+                                                'float_location': application_setting.float_location}
             return ApplicationSerializer.Query.reset_application(
                 {**ApplicationSerializer.ApplicationModel(application).data,
                  'stt_model_id': application.stt_model_id,
@@ -810,8 +827,8 @@ class ApplicationSerializer(serializers.Serializer):
             update_keys = ['name', 'desc', 'model_id', 'multiple_rounds_dialogue', 'prologue', 'status',
                            'dataset_setting', 'model_setting', 'problem_optimization', 'dialogue_number',
                            'stt_model_id', 'tts_model_id', 'tts_model_enable', 'stt_model_enable', 'tts_type',
-                           'api_key_is_active', 'icon', 'work_flow', 'model_params_setting','tts_model_params_setting',
+                           'api_key_is_active', 'icon', 'work_flow', 'model_params_setting', 'tts_model_params_setting',
-                           'problem_optimization_prompt']
+                           'problem_optimization_prompt', 'clean_time']
             for update_key in update_keys:
                 if update_key in instance and instance.get(update_key) is not None:
                     application.__setattr__(update_key, instance.get(update_key))
@@ -952,7 +969,8 @@ class ApplicationSerializer(serializers.Serializer):
             application_id = self.data.get('application_id')
             application = QuerySet(Application).filter(id=application_id).first()
             if application.tts_model_enable:
-                model = get_model_instance_by_model_user_id(application.tts_model_id, application.user_id, **application.tts_model_params_setting)
+                model = get_model_instance_by_model_user_id(application.tts_model_id, application.user_id,
+                                                            **application.tts_model_params_setting)
                 return model.text_to_speech(text)
 
     class ApplicationKeySerializerModel(serializers.ModelSerializer):

apps/common/job/__init__.py

@@ -7,7 +7,9 @@
     @desc:
 """
 from .client_access_num_job import *
+from .clean_chat_job import *
 
 
 def run():
     client_access_num_job.run()
+    clean_chat_job.run()

apps/common/job/clean_chat_job.py

@@ -0,0 +1,56 @@
+# coding=utf-8
+
+import logging
+import datetime
+
+from django.db import transaction
+from django.utils import timezone
+from apscheduler.schedulers.background import BackgroundScheduler
+from django_apscheduler.jobstores import DjangoJobStore
+from application.models import Application, Chat
+from django.db.models import Q
+from common.lock.impl.file_lock import FileLock
+
+scheduler = BackgroundScheduler()
+scheduler.add_jobstore(DjangoJobStore(), "default")
+lock = FileLock()
+
+
+def clean_chat_log_job():
+    logging.getLogger("max_kb").info('开始清理对话记录')
+    now = timezone.now()
+
+    applications = Application.objects.all().values('id', 'clean_time')
+    cutoff_dates = {
+        app['id']: now - datetime.timedelta(days=app['clean_time'] or 180)
+        for app in applications
+    }
+
+    query_conditions = Q()
+    for app_id, cutoff_date in cutoff_dates.items():
+        query_conditions |= Q(application_id=app_id, create_time__lt=cutoff_date)
+
+    batch_size = 500
+    while True:
+        with transaction.atomic():
+            logs_to_delete = Chat.objects.filter(query_conditions).values_list('id', flat=True)[:batch_size]
+            count = logs_to_delete.count()
+            if count == 0:
+                break
+            deleted_count, _ = Chat.objects.filter(id__in=logs_to_delete).delete()
+            if deleted_count < batch_size:
+                break
+
+    logging.getLogger("max_kb").info(f'结束清理对话记录')
+
+
+def run():
+    if lock.try_lock('clean_chat_log_job', 30 * 30):
+        try:
+            scheduler.start()
+            existing_job = scheduler.get_job(job_id='clean_chat_log')
+            if existing_job is not None:
+                existing_job.remove()
+            scheduler.add_job(clean_chat_log_job, 'cron', hour='0', minute='5', id='clean_chat_log')
+        finally:
+            lock.un_lock('clean_chat_log_job')

ui/src/views/log/index.vue

@@ -29,7 +29,10 @@
           style="margin-left: 10px"
           clearable
         />
-        <el-button class="float-right" @click="exportLog">导出</el-button>
+        <div style="display: flex; align-items: center" class="float-right">
+          <el-button @click="dialogVisible = true" type="primary">清除策略</el-button>
+          <el-button @click="exportLog">导出</el-button>
+        </div>
       </div>
 
       <app-table
@@ -145,6 +148,33 @@
       :next_disable="next_disable"
       @refresh="refresh"
     />
+    <el-dialog
+      title="清除策略"
+      v-model="dialogVisible"
+      width="25%"
+      :close-on-click-modal="false"
+      :close-on-press-escape="false"
+    >
+      <span>删除</span>
+      <el-input-number
+        v-model="days"
+        controls-position="right"
+        min="1"
+        max="100000"
+        style="width: 110px; margin-left: 8px; margin-right: 8px"
+      ></el-input-number>
+      <span>天之前的对话记录</span>
+      <template #footer>
+        <div class="dialog-footer" style="margin-top: 16px">
+          <el-button @click="dialogVisible = false">{{
+            $t('layout.topbar.avatar.dialog.cancel')
+          }}</el-button>
+          <el-button type="primary" @click="saveCleanTime">
+            {{ $t('layout.topbar.avatar.dialog.save') }}
+          </el-button>
+        </div>
+      </template>
+    </el-dialog>
   </LayoutContainer>
 </template>
 <script setup lang="ts">
@@ -203,6 +233,8 @@ const paginationConfig = reactive({
   page_size: 20,
   total: 0
 })
+const dialogVisible = ref(false)
+const days = ref<number>(180)
 const tableData = ref<any[]>([])
 const tableIndexMap = computed<Dict<number>>(() => {
   return tableData.value
@@ -355,9 +387,9 @@ function getList() {
 function getDetail() {
   application.asyncGetApplicationDetail(id as string, loading).then((res: any) => {
     detail.value = res.data
+    days.value = res.data.clean_time
   })
 }
-
 const exportLog = () => {
   const arr: string[] = []
   multipleSelection.value.map((v) => {
@@ -397,6 +429,21 @@ function changeDayHandle(val: number | string) {
   }
 }
 
+function saveCleanTime() {
+  const data = detail.value
+  data.clean_time = days.value
+  application
+    .asyncPutApplication(id as string, data, loading)
+    .then(() => {
+      MsgSuccess('保存成功')
+      dialogVisible.value = false
+      getDetail()
+    })
+    .catch(() => {
+      dialogVisible.value = false
+    })
+}
+
 onMounted(() => {
   changeDayHandle(history_day.value)
   getDetail()