refactor: 定时清理数据库中没关联应用的图片

apps/common/job/__init__.py

@@ -6,7 +6,6 @@
     @date：2024/3/14 11:54
     @desc:
 """
-from .clean_orphaned_file_job import *
 from .client_access_num_job import *
 from .clean_chat_job import *
 
@@ -14,4 +13,3 @@ from .clean_chat_job import *
 def run():
     client_access_num_job.run()
     clean_chat_job.run()
-    clean_orphaned_file_job.run()

apps/common/job/clean_chat_job.py

@@ -10,6 +10,7 @@ from django_apscheduler.jobstores import DjangoJobStore
 from application.models import Application, Chat
 from django.db.models import Q
 from common.lock.impl.file_lock import FileLock
+from dataset.models import File
 
 scheduler = BackgroundScheduler()
 scheduler.add_jobstore(DjangoJobStore(), "default")
@@ -38,6 +39,8 @@ def clean_chat_log_job():
             if count == 0:
                 break
             deleted_count, _ = Chat.objects.filter(id__in=logs_to_delete).delete()
+            # 删除对应的文件
+            File.objects.filter(~Q(meta__chat_id__in=logs_to_delete)).delete()
             if deleted_count < batch_size:
                 break
 

apps/common/job/clean_orphaned_file_job.py

@@ -1,40 +0,0 @@
-# coding=utf-8
-
-import logging
-
-from apscheduler.schedulers.background import BackgroundScheduler
-from django.db.models import Q
-from django_apscheduler.jobstores import DjangoJobStore
-
-from application.models import Chat
-from common.lock.impl.file_lock import FileLock
-from dataset.models import File
-
-scheduler = BackgroundScheduler()
-scheduler.add_jobstore(DjangoJobStore(), "default")
-lock = FileLock()
-
-
-def clean_debug_file():
-    logging.getLogger("max_kb").info('开始清理没有关联会话的上传文件')
-    existing_chat_ids = set(Chat.objects.values_list('id', flat=True))
-    # UUID to str
-    existing_chat_ids = [str(chat_id) for chat_id in existing_chat_ids]
-    print(existing_chat_ids)
-    # 查找引用的不存在的 chat_id 并删除相关记录
-    deleted_count, _ = File.objects.filter(~Q(meta__chat_id__in=existing_chat_ids)).delete()
-
-    logging.getLogger("max_kb").info(f'结束清理没有关联会话的上传文件: {deleted_count}')
-
-
-def run():
-    if lock.try_lock('clean_orphaned_file_job', 30 * 30):
-        try:
-            scheduler.start()
-            clean_orphaned_file = scheduler.get_job(job_id='clean_orphaned_file')
-            if clean_orphaned_file is not None:
-                clean_orphaned_file.remove()
-            scheduler.add_job(clean_debug_file, 'cron', hour='2', minute='0', second='0',
-                              id='clean_orphaned_file')
-        finally:
-            lock.un_lock('clean_orphaned_file_job')

apps/dataset/models/data_set.py

@@ -9,6 +9,8 @@
 import uuid
 
 from django.db import models
+from django.db.models.signals import pre_delete
+from django.dispatch import receiver
 
 from common.db.sql_execute import select_one
 from common.mixins.app_model_mixin import AppModelMixin
@@ -157,3 +159,9 @@ class File(AppModelMixin):
     def get_byte(self):
         result = select_one(f'SELECT lo_get({self.loid}) as "data"', [])
         return result['data']
+
+
+
+@receiver(pre_delete, sender=File)
+def on_delete_file(sender, instance, **kwargs):
+    select_one(f'SELECT lo_unlink({instance.loid})', [])