refactor: celery config

apps/knowledge/migrations/0002_document_paragraph_embedding_problem_and_more.py

@@ -0,0 +1,104 @@
+# Generated by Django 5.2 on 2025-04-29 03:28
+
+import django.contrib.postgres.search
+import django.db.models.deletion
+import knowledge.models.knowledge
+import uuid_utils.compat
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('knowledge', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Document',
+            fields=[
+                ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
+                ('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
+                ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
+                ('name', models.CharField(max_length=150, verbose_name='文档名称')),
+                ('char_length', models.IntegerField(verbose_name='文档字符数 冗余字段')),
+                ('status', models.CharField(default=knowledge.models.knowledge.Status.__str__, max_length=20, verbose_name='状态')),
+                ('status_meta', models.JSONField(default=knowledge.models.knowledge.default_status_meta, verbose_name='状态统计数据')),
+                ('is_active', models.BooleanField(default=True)),
+                ('type', models.IntegerField(choices=[(0, '通用类型'), (1, 'web站点类型'), (2, '飞书类型'), (3, '语雀类型')], default=0, verbose_name='类型')),
+                ('hit_handling_method', models.CharField(choices=[('optimization', '模型优化'), ('directly_return', '直接返回')], default='optimization', max_length=20, verbose_name='命中处理方式')),
+                ('directly_return_similarity', models.FloatField(default=0.9, verbose_name='直接回答相似度')),
+                ('meta', models.JSONField(default=dict, verbose_name='元数据')),
+                ('knowledge', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge', verbose_name='知识库id')),
+            ],
+            options={
+                'db_table': 'document',
+            },
+        ),
+        migrations.CreateModel(
+            name='Paragraph',
+            fields=[
+                ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
+                ('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
+                ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
+                ('content', models.CharField(max_length=102400, verbose_name='段落内容')),
+                ('title', models.CharField(default='', max_length=256, verbose_name='标题')),
+                ('status', models.CharField(default=knowledge.models.knowledge.Status.__str__, max_length=20, verbose_name='状态')),
+                ('status_meta', models.JSONField(default=knowledge.models.knowledge.default_status_meta, verbose_name='状态数据')),
+                ('hit_num', models.IntegerField(default=0, verbose_name='命中次数')),
+                ('is_active', models.BooleanField(default=True)),
+                ('document', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.document')),
+                ('knowledge', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge')),
+            ],
+            options={
+                'db_table': 'paragraph',
+            },
+        ),
+        migrations.CreateModel(
+            name='Embedding',
+            fields=[
+                ('id', models.CharField(max_length=128, primary_key=True, serialize=False, verbose_name='主键id')),
+                ('source_id', models.CharField(max_length=128, verbose_name='资源id')),
+                ('source_type', models.CharField(choices=[(0, '问题'), (1, '段落'), (2, '标题')], default=0, max_length=5, verbose_name='资源类型')),
+                ('is_active', models.BooleanField(default=True, max_length=1, verbose_name='是否可用')),
+                ('embedding', knowledge.models.knowledge.VectorField(verbose_name='向量')),
+                ('search_vector', django.contrib.postgres.search.SearchVectorField(default='', verbose_name='分词')),
+                ('meta', models.JSONField(default=dict, verbose_name='元数据')),
+                ('document', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.document', verbose_name='文档关联')),
+                ('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge', verbose_name='文档关联')),
+                ('paragraph', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.paragraph', verbose_name='段落关联')),
+            ],
+            options={
+                'db_table': 'embedding',
+            },
+        ),
+        migrations.CreateModel(
+            name='Problem',
+            fields=[
+                ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
+                ('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
+                ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
+                ('content', models.CharField(max_length=256, verbose_name='问题内容')),
+                ('hit_num', models.IntegerField(default=0, verbose_name='命中次数')),
+                ('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge')),
+            ],
+            options={
+                'db_table': 'problem',
+            },
+        ),
+        migrations.CreateModel(
+            name='ProblemParagraphMapping',
+            fields=[
+                ('create_time', models.DateTimeField(auto_now_add=True, verbose_name='创建时间')),
+                ('update_time', models.DateTimeField(auto_now=True, verbose_name='修改时间')),
+                ('id', models.UUIDField(default=uuid_utils.compat.uuid7, editable=False, primary_key=True, serialize=False, verbose_name='主键id')),
+                ('document', models.ForeignKey(on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.document')),
+                ('knowledge', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.knowledge')),
+                ('paragraph', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.paragraph')),
+                ('problem', models.ForeignKey(db_constraint=False, on_delete=django.db.models.deletion.DO_NOTHING, to='knowledge.problem')),
+            ],
+            options={
+                'db_table': 'problem_paragraph_mapping',
+            },
+        ),
+    ]

apps/maxkb/settings/__init__.py

@@ -7,4 +7,6 @@
     @desc:
 """
 from .base import *
+from .logging import *
 from .auth import *
+from .lib import *

apps/maxkb/settings/base.py

@@ -44,6 +44,7 @@ INSTALLED_APPS = [
     'common',
     'system_manage',
     'models_provider',
+    'django_celery_beat'
 ]
 
 MIDDLEWARE = [

apps/maxkb/settings/lib.py

@@ -0,0 +1,48 @@
+# coding=utf-8
+"""
+    @project: MaxKB
+    @Author：虎
+    @file： lib.py
+    @date：2024/8/16 17:12
+    @desc:
+"""
+import os
+import shutil
+
+from maxkb.const import CONFIG, PROJECT_DIR
+
+# celery相关配置
+celery_data_dir = os.path.join(PROJECT_DIR, 'data', 'celery_task')
+if not os.path.exists(celery_data_dir) or not os.path.isdir(celery_data_dir):
+    os.makedirs(celery_data_dir)
+broker_path = os.path.join(celery_data_dir, "celery_db.sqlite3")
+backend_path = os.path.join(celery_data_dir, "celery_results.sqlite3")
+# 使用sql_lite 当做broker 和 响应接收
+CELERY_BROKER_URL = f'sqla+sqlite:///{broker_path}'
+CELERY_result_backend = f'db+sqlite:///{backend_path}'
+CELERY_timezone = CONFIG.TIME_ZONE
+CELERY_ENABLE_UTC = False
+CELERY_task_serializer = 'pickle'
+CELERY_result_serializer = 'pickle'
+CELERY_accept_content = ['json', 'pickle']
+CELERY_RESULT_EXPIRES = 600
+CELERY_WORKER_TASK_LOG_FORMAT = '%(asctime).19s %(message)s'
+CELERY_WORKER_LOG_FORMAT = '%(asctime).19s %(message)s'
+CELERY_TASK_EAGER_PROPAGATES = True
+CELERY_WORKER_REDIRECT_STDOUTS = True
+CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = "INFO"
+CELERY_TASK_SOFT_TIME_LIMIT = 3600
+CELERY_WORKER_CANCEL_LONG_RUNNING_TASKS_ON_CONNECTION_LOSS = True
+CELERY_ACKS_LATE = True
+celery_once_path = os.path.join(celery_data_dir, "celery_once")
+try:
+    if os.path.exists(celery_once_path) and os.path.isdir(celery_once_path):
+        shutil.rmtree(celery_once_path)
+except Exception as e:
+    pass
+CELERY_ONCE = {
+    'backend': 'celery_once.backends.File',
+    'settings': {'location': celery_once_path}
+}
+CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
+CELERY_LOG_DIR = os.path.join(PROJECT_DIR, 'logs', 'celery')

apps/maxkb/settings/logging.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+#
+import os
+
+from ..const import PROJECT_DIR, CONFIG
+
+LOG_DIR = os.path.join(PROJECT_DIR, 'data', 'logs')
+MAX_KB_LOG_FILE = os.path.join(LOG_DIR, 'max_kb.log')
+DRF_EXCEPTION_LOG_FILE = os.path.join(LOG_DIR, 'drf_exception.log')
+UNEXPECTED_EXCEPTION_LOG_FILE = os.path.join(LOG_DIR, 'unexpected_exception.log')
+LOG_LEVEL = "DEBUG"
+
+LOGGING = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'formatters': {
+        'verbose': {
+            'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
+        },
+        'main': {
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'format': '%(asctime)s [%(module)s %(levelname)s] %(message)s',
+        },
+        'exception': {
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+            'format': '\n%(asctime)s [%(levelname)s] %(message)s',
+        },
+        'simple': {
+            'format': '%(levelname)s %(message)s'
+        },
+        'syslog': {
+            'format': 'jumpserver: %(message)s'
+        },
+        'msg': {
+            'format': '%(message)s'
+        }
+    },
+    'handlers': {
+        'null': {
+            'level': 'DEBUG',
+            'class': 'logging.NullHandler',
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'main'
+        },
+        'file': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'maxBytes': 1024 * 1024 * 100,
+            'backupCount': 7,
+            'formatter': 'main',
+            'filename': MAX_KB_LOG_FILE,
+        },
+        'drf_exception': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'formatter': 'exception',
+            'maxBytes': 1024 * 1024 * 100,
+            'backupCount': 7,
+            'filename': DRF_EXCEPTION_LOG_FILE,
+        },
+        'unexpected_exception': {
+            'encoding': 'utf8',
+            'level': 'DEBUG',
+            'class': 'logging.handlers.RotatingFileHandler',
+            'formatter': 'exception',
+            'maxBytes': 1024 * 1024 * 100,
+            'backupCount': 7,
+            'filename': UNEXPECTED_EXCEPTION_LOG_FILE,
+        },
+        'syslog': {
+            'level': 'INFO',
+            'class': 'logging.NullHandler',
+            'formatter': 'syslog'
+        },
+    },
+    'loggers': {
+        'django': {
+            'handlers': ['null'],
+            'propagate': False,
+            'level': LOG_LEVEL,
+        },
+        'django.request': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'sqlalchemy': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': "ERROR",
+            'propagate': False,
+        },
+        'django.db.backends': {
+            'handlers': ['console', 'file', 'syslog'],
+            'propagate': False,
+            'level': LOG_LEVEL,
+        },
+        'django.server': {
+            'handlers': ['console', 'file', 'syslog'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'max_kb_error': {
+            'handlers': ['console', 'unexpected_exception'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'max_kb': {
+            'handlers': ['console', 'file'],
+            'level': LOG_LEVEL,
+            'propagate': False,
+        },
+        'common.event': {
+            'handlers': ['console', 'file'],
+            'level': "DEBUG",
+            'propagate': False,
+        },
+    }
+}
+
+SYSLOG_ENABLE = CONFIG.SYSLOG_ENABLE
+
+if not os.path.isdir(LOG_DIR):
+    os.makedirs(LOG_DIR, mode=0o755)