fix: Scheduled file scheduled deletion (#3613)

apps/common/job/clean_chat_job.py

@@ -9,7 +9,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from django_apscheduler.jobstores import DjangoJobStore
 from application.models import Application, Chat, ChatRecord
 from django.db.models import Q, Max
-from common.utils.lock import try_lock, un_lock
+from common.utils.lock import try_lock, un_lock, lock
 from common.utils.logger import maxkb_logger
 
 from knowledge.models import File
@@ -19,6 +19,11 @@ scheduler.add_jobstore(DjangoJobStore(), "default")
 
 
 def clean_chat_log_job():
+    clean_chat_log_job_lock()
+
+
+@lock(lock_key='clean_chat_log_job', timeout=30)
+def clean_chat_log_job_lock():
     from django.utils.translation import gettext_lazy as _
     maxkb_logger.info(_('start clean chat log'))
     now = timezone.now()

apps/common/job/clean_debug_file_job.py

@@ -1,6 +1,5 @@
 # coding=utf-8
 
-import logging
 from datetime import timedelta
 
 from apscheduler.schedulers.background import BackgroundScheduler
@@ -8,30 +7,40 @@ from django.db.models import Q
 from django.utils import timezone
 from django_apscheduler.jobstores import DjangoJobStore
 
-from common.utils.lock import un_lock, try_lock
+from common.utils.lock import un_lock, try_lock, lock
 from common.utils.logger import maxkb_logger
-from knowledge.models import File
+from knowledge.models import File, FileSourceType
 
 scheduler = BackgroundScheduler()
 scheduler.add_jobstore(DjangoJobStore(), "default")
 
 
 def clean_debug_file():
+    clean_debug_file_lock()
+
+
+@lock(lock_key='clean_debug_file', timeout=30)
+def clean_debug_file_lock():
     from django.utils.translation import gettext_lazy as _
     maxkb_logger.info(_('start clean debug file'))
+    minutes_30_ago = timezone.now() - timedelta(minutes=30)
     two_hours_ago = timezone.now() - timedelta(hours=2)
+    one_days_ago = timezone.now() - timedelta(hours=24)
     # 删除对应的文件
-    File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete()
+    File.objects.filter(
+        Q(create_time__lt=one_days_ago, source_type=FileSourceType.TEMPORARY_1_DAY.value) |
+        Q(create_time__lt=two_hours_ago, source_type=FileSourceType.TEMPORARY_120_MINUTE.value) |
+        Q(create_time__lt=minutes_30_ago, source_type=FileSourceType.TEMPORARY_30_MINUTE.value)).delete()
     maxkb_logger.info(_('end clean debug file'))
 
 
 def run():
-    if try_lock('clean_debug_file', 30 * 30):
+    if try_lock('clean_debug_file', 30):
         try:
             scheduler.start()
             clean_debug_file_job = scheduler.get_job(job_id='clean_debug_file')
             if clean_debug_file_job is not None:
                 clean_debug_file_job.remove()
-            scheduler.add_job(clean_debug_file, 'cron', hour='2', minute='0', second='0', id='clean_debug_file')
+            scheduler.add_job(clean_debug_file, 'cron', hour='*', minute='*/30', second='0', id='clean_debug_file')
         finally:
             un_lock('clean_debug_file')

apps/common/utils/lock.py

@@ -34,18 +34,20 @@ def un_lock(key: str):
     return memory_cache.delete(key)
 
 
-def lock(lock_key):
+def lock(lock_key, timeout=None):
     """
     给一个函数上锁
-    :param lock_key: 上锁key 字符串|函数  函数返回值为字符串
+    @param lock_key: 上锁key 字符串|函数  函数返回值为字符串
+    @param timeout:  超时时间
     :return: 装饰器函数 当前装饰器主要限制一个key只能一个线程去调用 相同key只能阻塞等待上一个任务执行完毕 不同key不需要等待
+
     """
 
     def inner(func):
         def run(*args, **kwargs):
             key = lock_key(*args, **kwargs) if callable(lock_key) else lock_key
             try:
-                if try_lock(key=key):
+                if try_lock(key=key, timeout=timeout):
                     return func(*args, **kwargs)
             finally:
                 un_lock(key=key)