zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: replace logging with maxkb_logger for consistent logging across modules

Browse Source
This commit is contained in:
CaptainB 2025-06-25 16:46:38 +08:00
parent fe066aea68
commit fe8f87834d
19 changed files with 79 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py
View File

@ -25,6 +25,7 @@ from application.flow.step_node.ai_chat_step_node.i_chat_node import IChatNode
from application.flow.tools import Reasoning from application.flow.tools import Reasoning
from models_provider.models import Model from models_provider.models import Model
from models_provider.tools import get_model_credential, get_model_instance_by_model_workspace_id from models_provider.tools import get_model_credential, get_model_instance_by_model_workspace_id
from common.utils.logger import maxkb_logger
tool_message_template = """ tool_message_template = """
<details> <details>
@ -126,7 +127,7 @@ def mcp_response_generator(chat_model, message_list, mcp_servers):
except StopAsyncIteration: except StopAsyncIteration:
break break
except Exception as e: except Exception as e:
logging.getLogger("max_kb").error(f'Exception: {e}') maxkb_logger.error(f'Exception: {e}')
finally: finally:
loop.close() loop.close()

33
apps/common/event/listener_manage.py
View File

@ -6,7 +6,6 @@
@date：2023/10/20 14:01 @date：2023/10/20 14:01
@desc: @desc:
""" """
import logging
import os import os
import threading import threading
import datetime import datetime
@ -22,13 +21,13 @@ from common.config.embedding_config import VectorStore
from common.db.search import native_search, get_dynamics_model, native_update from common.db.search import native_search, get_dynamics_model, native_update
from common.utils.common import get_file_content from common.utils.common import get_file_content
from common.utils.lock import try_lock, un_lock from common.utils.lock import try_lock, un_lock
from common.utils.logger import maxkb_logger, maxkb_error_logger
from common.utils.page_utils import page_desc from common.utils.page_utils import page_desc
from knowledge.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State,SourceType, SearchMode from knowledge.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State,SourceType, SearchMode
from maxkb.conf import (PROJECT_DIR) from maxkb.conf import (PROJECT_DIR)
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
max_kb_error = logging.getLogger(__file__)
max_kb = logging.getLogger(__file__)
lock = threading.Lock() lock = threading.Lock()
@ -87,12 +86,12 @@ class ListenerManagement:
ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list=paragraph_id_list, ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list=paragraph_id_list,
embedding_model=embedding_model) embedding_model=embedding_model)
except Exception as e: except Exception as e:
max_kb_error.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format( maxkb_error_logger.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format(
paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc())) paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
@staticmethod @staticmethod
def embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model: Embeddings): def embedding_by_paragraph_data_list(data_list, paragraph_id_list, embedding_model: Embeddings):
max_kb.info(_('Start--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list)) maxkb_logger.info(_('Start--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list))
status = Status.success status = Status.success
try: try:
# 删除段落 # 删除段落
@ -104,12 +103,12 @@ class ListenerManagement:
# 批量向量化 # 批量向量化
VectorStore.get_embedding_vector().batch_save(data_list, embedding_model, is_save_function) VectorStore.get_embedding_vector().batch_save(data_list, embedding_model, is_save_function)
except Exception as e: except Exception as e:
max_kb_error.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format( maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format(
paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc())) paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
status = Status.error status = Status.error
finally: finally:
QuerySet(Paragraph).filter(id__in=paragraph_id_list).update(**{'status': status}) QuerySet(Paragraph).filter(id__in=paragraph_id_list).update(**{'status': status})
max_kb.info( maxkb_logger.info(
_('End--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list)) _('End--->Embedding paragraph: {paragraph_id_list}').format(paragraph_id_list=paragraph_id_list))
@staticmethod @staticmethod
@ -119,7 +118,7 @@ class ListenerManagement:
@param paragraph_id: 段落id @param paragraph_id: 段落id
@param embedding_model: 向量模型 @param embedding_model: 向量模型
""" """
max_kb.info(_('Start--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id)) maxkb_logger.info(_('Start--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id))
# 更新到开始状态 # 更新到开始状态
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, State.STARTED) ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, State.STARTED)
try: try:
@ -144,12 +143,12 @@ class ListenerManagement:
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
State.SUCCESS) State.SUCCESS)
except Exception as e: except Exception as e:
max_kb_error.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format( maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format(
paragraph_id=paragraph_id, error=str(e), traceback=traceback.format_exc())) paragraph_id=paragraph_id, error=str(e), traceback=traceback.format_exc()))
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING, ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
State.FAILURE) State.FAILURE)
finally: finally:
max_kb.info(_('End--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id)) maxkb_logger.info(_('End--->Embedding paragraph: {paragraph_id}').format(paragraph_id=paragraph_id))
@staticmethod @staticmethod
def embedding_by_data_list(data_list: List, embedding_model: Embeddings): def embedding_by_data_list(data_list: List, embedding_model: Embeddings):
@ -265,7 +264,7 @@ class ListenerManagement:
if is_the_task_interrupted(): if is_the_task_interrupted():
return return
max_kb.info(_('Start--->Embedding document: {document_id}').format(document_id=document_id) maxkb_logger.info(_('Start--->Embedding document: {document_id}').format(document_id=document_id)
) )
# 批量修改状态为PADDING # 批量修改状态为PADDING
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING, ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
@ -285,12 +284,12 @@ class ListenerManagement:
document_id)), document_id)),
is_the_task_interrupted) is_the_task_interrupted)
except Exception as e: except Exception as e:
max_kb_error.error(_('Vectorized document: {document_id} error {error} {traceback}').format( maxkb_error_logger.error(_('Vectorized document: {document_id} error {error} {traceback}').format(
document_id=document_id, error=str(e), traceback=traceback.format_exc())) document_id=document_id, error=str(e), traceback=traceback.format_exc()))
finally: finally:
ListenerManagement.post_update_document_status(document_id, TaskType.EMBEDDING) ListenerManagement.post_update_document_status(document_id, TaskType.EMBEDDING)
ListenerManagement.get_aggregation_document_status(document_id)() ListenerManagement.get_aggregation_document_status(document_id)()
max_kb.info(_('End--->Embedding document: {document_id}').format(document_id=document_id)) maxkb_logger.info(_('End--->Embedding document: {document_id}').format(document_id=document_id))
un_lock('embedding' + str(document_id)) un_lock('embedding' + str(document_id))
@staticmethod @staticmethod
@ -301,18 +300,18 @@ class ListenerManagement:
@param embedding_model 向量模型 @param embedding_model 向量模型
:return: None :return: None
""" """
max_kb.info(_('Start--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) maxkb_logger.info(_('Start--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
try: try:
ListenerManagement.delete_embedding_by_knowledge(knowledge_id) ListenerManagement.delete_embedding_by_knowledge(knowledge_id)
document_list = QuerySet(Document).filter(knowledge_id=knowledge_id) document_list = QuerySet(Document).filter(knowledge_id=knowledge_id)
max_kb.info(_('Start--->Embedding document: {document_list}').format(document_list=document_list)) maxkb_logger.info(_('Start--->Embedding document: {document_list}').format(document_list=document_list))
for document in document_list: for document in document_list:
ListenerManagement.embedding_by_document(document.id, embedding_model=embedding_model) ListenerManagement.embedding_by_document(document.id, embedding_model=embedding_model)
except Exception as e: except Exception as e:
max_kb_error.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format( maxkb_error_logger.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format(
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc())) knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
finally: finally:
max_kb.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) maxkb_logger.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
@staticmethod @staticmethod
def delete_embedding_by_document(document_id): def delete_embedding_by_document(document_id):

7
apps/common/handle/impl/table/csv_parse_table_handle.py
View File

@ -4,8 +4,7 @@ import logging
from charset_normalizer import detect from charset_normalizer import detect
from common.handle.base_parse_table_handle import BaseParseTableHandle from common.handle.base_parse_table_handle import BaseParseTableHandle
from common.utils.logger import maxkb_logger, maxkb_error_logger
max_kb = logging.getLogger("max_kb")
class CsvParseTableHandle(BaseParseTableHandle): class CsvParseTableHandle(BaseParseTableHandle):
@ -20,7 +19,7 @@ class CsvParseTableHandle(BaseParseTableHandle):
try: try:
content = buffer.decode(detect(buffer)['encoding']) content = buffer.decode(detect(buffer)['encoding'])
except BaseException as e: except BaseException as e:
max_kb.error(f'csv split handle error: {e}') maxkb_logger.error(f'csv split handle error: {e}')
return [{'name': file.name, 'paragraphs': []}] return [{'name': file.name, 'paragraphs': []}]
csv_model = content.split('\n') csv_model = content.split('\n')
@ -40,5 +39,5 @@ class CsvParseTableHandle(BaseParseTableHandle):
try: try:
return buffer.decode(detect(buffer)['encoding']) return buffer.decode(detect(buffer)['encoding'])
except BaseException as e: except BaseException as e:
max_kb.error(f'csv split handle error: {e}') maxkb_error_logger.error(f'csv split handle error: {e}')
return f'error: {e}' return f'error: {e}'

5
apps/common/handle/impl/table/xls_parse_table_handle.py
View File

@ -4,8 +4,7 @@ import logging
import xlrd import xlrd
from common.handle.base_parse_table_handle import BaseParseTableHandle from common.handle.base_parse_table_handle import BaseParseTableHandle
from common.utils.logger import maxkb_error_logger
max_kb = logging.getLogger("max_kb")
class XlsParseTableHandle(BaseParseTableHandle): class XlsParseTableHandle(BaseParseTableHandle):
@ -56,7 +55,7 @@ class XlsParseTableHandle(BaseParseTableHandle):
result.append({'name': sheet.name, 'paragraphs': paragraphs}) result.append({'name': sheet.name, 'paragraphs': paragraphs})
except BaseException as e: except BaseException as e:
max_kb.error(f'excel split handle error: {e}') maxkb_error_logger.error(f'excel split handle error: {e}')
return [{'name': file.name, 'paragraphs': []}] return [{'name': file.name, 'paragraphs': []}]
return result return result

7
apps/common/handle/impl/table/xlsx_parse_table_handle.py
View File

@ -6,8 +6,7 @@ from openpyxl import load_workbook
from common.handle.base_parse_table_handle import BaseParseTableHandle from common.handle.base_parse_table_handle import BaseParseTableHandle
from common.handle.impl.common_handle import xlsx_embed_cells_images from common.handle.impl.common_handle import xlsx_embed_cells_images
from common.utils.logger import maxkb_error_logger
max_kb = logging.getLogger("max_kb")
class XlsxParseTableHandle(BaseParseTableHandle): class XlsxParseTableHandle(BaseParseTableHandle):
@ -74,7 +73,7 @@ class XlsxParseTableHandle(BaseParseTableHandle):
result.append({'name': sheetname, 'paragraphs': paragraphs}) result.append({'name': sheetname, 'paragraphs': paragraphs})
except BaseException as e: except BaseException as e:
max_kb.error(f'excel split handle error: {e}') maxkb_error_logger.error(f'excel split handle error: {e}')
return [{'name': file.name, 'paragraphs': []}] return [{'name': file.name, 'paragraphs': []}]
return result return result
@ -87,7 +86,7 @@ class XlsxParseTableHandle(BaseParseTableHandle):
if len(image_dict) > 0: if len(image_dict) > 0:
save_image(image_dict.values()) save_image(image_dict.values())
except Exception as e: except Exception as e:
logging.getLogger("max_kb").error(f'Exception: {e}') maxkb_error_logger.error(f'Exception: {e}')
image_dict = {} image_dict = {}
md_tables = '' md_tables = ''
# 如果未指定 sheet_name，则使用第一个工作表 # 如果未指定 sheet_name，则使用第一个工作表

10
apps/common/handle/impl/text/pdf_split_handle.py
View File

@ -19,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
from langchain_community.document_loaders import PyPDFLoader from langchain_community.document_loaders import PyPDFLoader
from common.handle.base_split_handle import BaseSplitHandle from common.handle.base_split_handle import BaseSplitHandle
from common.utils.logger import maxkb_error_logger, maxkb_logger
from common.utils.split_model import SplitModel from common.utils.split_model import SplitModel
default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'), default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'),
@ -29,7 +30,6 @@ default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'),
re.compile("(?<=\\n)(?<!#)###### (?!#).*|(?<=^)(?<!#)###### (?!#).*"), re.compile("(?<=\\n)(?<!#)###### (?!#).*|(?<=^)(?<!#)###### (?!#).*"),
re.compile("(?<!\n)\n\n+")] re.compile("(?<!\n)\n\n+")]
max_kb = logging.getLogger("max_kb")
def check_links_in_pdf(doc): def check_links_in_pdf(doc):
@ -72,7 +72,7 @@ class PdfSplitHandle(BaseSplitHandle):
else: else:
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit) split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
except BaseException as e: except BaseException as e:
max_kb.error(f"File: {file.name}, error: {e}") maxkb_error_logger.error(f"File: {file.name}, error: {e}")
return {'name': file.name, return {'name': file.name,
'content': []} 'content': []}
finally: finally:
@ -109,7 +109,7 @@ class PdfSplitHandle(BaseSplitHandle):
raise e raise e
except BaseException as e: except BaseException as e:
# 当页出错继续进行下一页，防止一个页面出错导致整个文件解析失败 # 当页出错继续进行下一页，防止一个页面出错导致整个文件解析失败
max_kb.error(f"File: {file.name}, Page: {page_num + 1}, error: {e}") maxkb_error_logger.error(f"File: {file.name}, Page: {page_num + 1}, error: {e}")
continue continue
finally: finally:
os.remove(page_num_pdf) os.remove(page_num_pdf)
@ -120,7 +120,7 @@ class PdfSplitHandle(BaseSplitHandle):
content = content.replace('\0', '') content = content.replace('\0', '')
elapsed_time = time.time() - start_time elapsed_time = time.time() - start_time
max_kb.debug( maxkb_logger.debug(
f"File: {file.name}, Page: {page_num + 1}, Time : {elapsed_time: .3f}s, content-length: {len(page_content)}") f"File: {file.name}, Page: {page_num + 1}, Time : {elapsed_time: .3f}s, content-length: {len(page_content)}")
return content return content
@ -278,7 +278,7 @@ class PdfSplitHandle(BaseSplitHandle):
pre_toc[i]['content'] = re.sub(r'(?<!。)\n+', '', pre_toc[i]['content']) pre_toc[i]['content'] = re.sub(r'(?<!。)\n+', '', pre_toc[i]['content'])
pre_toc[i]['content'] = re.sub(r'(?<!.)\n+', '', pre_toc[i]['content']) pre_toc[i]['content'] = re.sub(r'(?<!.)\n+', '', pre_toc[i]['content'])
except BaseException as e: except BaseException as e:
max_kb.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e)) maxkb_error_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
if pattern_list is not None and len(pattern_list) > 0: if pattern_list is not None and len(pattern_list) > 0:
split_model = SplitModel(pattern_list, with_filter, limit) split_model = SplitModel(pattern_list, with_filter, limit)
else: else:

5
apps/common/job/clean_chat_job.py
View File

@ -10,6 +10,7 @@ from django_apscheduler.jobstores import DjangoJobStore
from application.models import Application, Chat, ChatRecord from application.models import Application, Chat, ChatRecord
from django.db.models import Q, Max from django.db.models import Q, Max
from common.lock.impl.file_lock import FileLock from common.lock.impl.file_lock import FileLock
from common.utils.logger import maxkb_logger
from knowledge.models import File from knowledge.models import File
@ -20,7 +21,7 @@ lock = FileLock()
def clean_chat_log_job(): def clean_chat_log_job():
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
logging.getLogger("max_kb").info(_('start clean chat log')) maxkb_logger.info(_('start clean chat log'))
now = timezone.now() now = timezone.now()
applications = Application.objects.all().values('id', 'clean_time') applications = Application.objects.all().values('id', 'clean_time')
@ -66,7 +67,7 @@ def clean_chat_log_job():
if deleted_count < batch_size: if deleted_count < batch_size:
break break
logging.getLogger("max_kb").info(_('end clean chat log')) maxkb_logger.info(_('end clean chat log'))
def run(): def run():

5
apps/common/job/clean_debug_file_job.py
View File

@ -9,6 +9,7 @@ from django.utils import timezone
from django_apscheduler.jobstores import DjangoJobStore from django_apscheduler.jobstores import DjangoJobStore
from common.lock.impl.file_lock import FileLock from common.lock.impl.file_lock import FileLock
from common.utils.logger import maxkb_logger
from knowledge.models import File from knowledge.models import File
scheduler = BackgroundScheduler() scheduler = BackgroundScheduler()
@ -18,11 +19,11 @@ lock = FileLock()
def clean_debug_file(): def clean_debug_file():
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
logging.getLogger("max_kb").info(_('start clean debug file')) maxkb_logger.info(_('start clean debug file'))
two_hours_ago = timezone.now() - timedelta(hours=2) two_hours_ago = timezone.now() - timedelta(hours=2)
# 删除对应的文件 # 删除对应的文件
File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete() File.objects.filter(Q(create_time__lt=two_hours_ago) & Q(meta__debug=True)).delete()
logging.getLogger("max_kb").info(_('end clean debug file')) maxkb_logger.info(_('end clean debug file'))
def run(): def run():

12
apps/common/utils/fork.py
View File

@ -10,6 +10,8 @@ import html2text as ht
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from common.utils.logger import maxkb_error_logger, maxkb_logger
requests.packages.urllib3.disable_warnings() requests.packages.urllib3.disable_warnings()
@ -145,7 +147,7 @@ class Fork:
try: try:
html_content = response.content.decode(charset) html_content = response.content.decode(charset)
except Exception as e: except Exception as e:
logging.getLogger("max_kb").error(f'{e}') maxkb_error_logger.error(f'{e}')
return BeautifulSoup(html_content, "html.parser") return BeautifulSoup(html_content, "html.parser")
return beautiful_soup return beautiful_soup
@ -156,14 +158,14 @@ class Fork:
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36' 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36'
} }
logging.getLogger("max_kb").info(f'fork:{self.base_fork_url}') maxkb_logger.info(f'fork:{self.base_fork_url}')
response = requests.get(self.base_fork_url, verify=False, headers=headers) response = requests.get(self.base_fork_url, verify=False, headers=headers)
if response.status_code != 200: if response.status_code != 200:
logging.getLogger("max_kb").error(f"url: {self.base_fork_url} code:{response.status_code}") maxkb_error_logger.error(f"url: {self.base_fork_url} code:{response.status_code}")
return Fork.Response.error(f"url: {self.base_fork_url} code:{response.status_code}") return Fork.Response.error(f"url: {self.base_fork_url} code:{response.status_code}")
bf = self.get_beautiful_soup(response) bf = self.get_beautiful_soup(response)
except Exception as e: except Exception as e:
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}') maxkb_error_logger.error(f'{str(e)}:{traceback.format_exc()}')
return Fork.Response.error(str(e)) return Fork.Response.error(str(e))
bf = self.reset_beautiful_soup(bf) bf = self.reset_beautiful_soup(bf)
link_list = self.get_child_link_list(bf) link_list = self.get_child_link_list(bf)
@ -173,6 +175,6 @@ class Fork:
def handler(base_url, response: Fork.Response): def handler(base_url, response: Fork.Response):
logging.getLogger("max_kb").info(base_url.url, base_url.tag.text if base_url.tag else None, response.content) maxkb_logger.info(base_url.url, base_url.tag.text if base_url.tag else None, response.content)
# ForkManage('https://bbs.fit2cloud.com/c/de/6', ['.md-content']).fork(3, set(), handler) # ForkManage('https://bbs.fit2cloud.com/c/de/6', ['.md-content']).fork(3, set(), handler)

3
apps/knowledge/serializers/knowledge.py
View File

@ -30,6 +30,7 @@ from common.event import ListenerManagement
from common.exception.app_exception import AppApiException from common.exception.app_exception import AppApiException
from common.utils.common import valid_license, post, get_file_content, parse_image from common.utils.common import valid_license, post, get_file_content, parse_image
from common.utils.fork import Fork, ChildLink from common.utils.fork import Fork, ChildLink
from common.utils.logger import maxkb_logger
from common.utils.split_model import get_split_model from common.utils.split_model import get_split_model
from knowledge.models import Knowledge, KnowledgeScope, KnowledgeType, Document, Paragraph, Problem, \ from knowledge.models import Knowledge, KnowledgeScope, KnowledgeType, Document, Paragraph, Problem, \
ProblemParagraphMapping, TaskType, State, SearchMode, KnowledgeFolder ProblemParagraphMapping, TaskType, State, SearchMode, KnowledgeFolder
@ -630,7 +631,7 @@ class KnowledgeSerializer(serializers.Serializer):
document_name = child_link.tag.text if child_link.tag is not None and len( document_name = child_link.tag.text if child_link.tag is not None and len(
child_link.tag.text.strip()) > 0 else child_link.url child_link.tag.text.strip()) > 0 else child_link.url
paragraphs = get_split_model('web.md').parse(response.content) paragraphs = get_split_model('web.md').parse(response.content)
logging.getLogger("max_kb").info(child_link.url.strip()) maxkb_logger.info(child_link.url.strip())
first = QuerySet(Document).filter( first = QuerySet(Document).filter(
meta__source_url=child_link.url.strip(), meta__source_url=child_link.url.strip(),
knowledge=knowledge knowledge=knowledge

15
apps/knowledge/task/embedding.py
View File

@ -11,16 +11,15 @@ from django.utils.translation import gettext_lazy as _
from common.config.embedding_config import ModelManage from common.config.embedding_config import ModelManage
from common.event import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \ from common.event import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \
UpdateEmbeddingDocumentIdArgs UpdateEmbeddingDocumentIdArgs
from common.utils.logger import maxkb_error_logger, maxkb_logger
from knowledge.models import Document, TaskType, State from knowledge.models import Document, TaskType, State
from models_provider.tools import get_model from models_provider.tools import get_model
from models_provider.models import Model from models_provider.models import Model
from ops import celery_app from ops import celery_app
max_kb_error = logging.getLogger("max_kb_error")
max_kb = logging.getLogger("max_kb")
def get_embedding_model(model_id, exception_handler=lambda e: max_kb_error.error( def get_embedding_model(model_id, exception_handler=lambda e: maxkb_error_logger.error(
_('Failed to obtain vector model: {error} {traceback}').format( _('Failed to obtain vector model: {error} {traceback}').format(
error=str(e), error=str(e),
traceback=traceback.format_exc() traceback=traceback.format_exc()
@ -70,7 +69,7 @@ def embedding_by_document(document_id, model_id, state_list=None):
def exception_handler(e): def exception_handler(e):
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING, ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
State.FAILURE) State.FAILURE)
max_kb_error.error( maxkb_error_logger.error(
_('Failed to obtain vector model: {error} {traceback}').format( _('Failed to obtain vector model: {error} {traceback}').format(
error=str(e), error=str(e),
traceback=traceback.format_exc() traceback=traceback.format_exc()
@ -100,11 +99,11 @@ def embedding_by_knowledge(knowledge_id, model_id):
@param model_id 向量模型 @param model_id 向量模型
:return: None :return: None
""" """
max_kb.info(_('Start--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) maxkb_logger.info(_('Start--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
try: try:
ListenerManagement.delete_embedding_by_knowledge(knowledge_id) ListenerManagement.delete_embedding_by_knowledge(knowledge_id)
document_list = QuerySet(Document).filter(knowledge_id=knowledge_id) document_list = QuerySet(Document).filter(knowledge_id=knowledge_id)
max_kb.info(_('Knowledge documentation: {document_names}').format( maxkb_logger.info(_('Knowledge documentation: {document_names}').format(
document_names=", ".join([d.name for d in document_list]))) document_names=", ".join([d.name for d in document_list])))
for document in document_list: for document in document_list:
try: try:
@ -112,12 +111,12 @@ def embedding_by_knowledge(knowledge_id, model_id):
except Exception as e: except Exception as e:
pass pass
except Exception as e: except Exception as e:
max_kb_error.error( maxkb_error_logger.error(
_('Vectorized knowledge: {knowledge_id} error {error} {traceback}'.format(knowledge_id=knowledge_id, _('Vectorized knowledge: {knowledge_id} error {error} {traceback}'.format(knowledge_id=knowledge_id,
error=str(e), error=str(e),
traceback=traceback.format_exc()))) traceback=traceback.format_exc())))
finally: finally:
max_kb.info(_('End--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id)) maxkb_logger.info(_('End--->Vectorized knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
def embedding_by_problem(args, model_id): def embedding_by_problem(args, model_id):

10
apps/knowledge/task/generate.py
View File

@ -9,6 +9,7 @@ from langchain_core.messages import HumanMessage
from common.config.embedding_config import ModelManage from common.config.embedding_config import ModelManage
from common.event import ListenerManagement from common.event import ListenerManagement
from common.utils.logger import maxkb_error_logger, maxkb_logger
from common.utils.page_utils import page, page_desc from common.utils.page_utils import page, page_desc
from knowledge.models import Paragraph, Document, Status, TaskType, State from knowledge.models import Paragraph, Document, Status, TaskType, State
from knowledge.task.handler import save_problem from knowledge.task.handler import save_problem
@ -16,9 +17,6 @@ from models_provider.models import Model
from models_provider.tools import get_model from models_provider.tools import get_model
from ops import celery_app from ops import celery_app
max_kb_error = logging.getLogger("max_kb_error")
max_kb = logging.getLogger("max_kb")
def get_llm_model(model_id): def get_llm_model(model_id):
model = QuerySet(Model).filter(id=model_id).first() model = QuerySet(Model).filter(id=model_id).first()
@ -102,12 +100,12 @@ def generate_related_by_document_id(document_id, model_id, prompt, state_list=No
).filter(task_type_status__in=state_list, document_id=document_id) ).filter(task_type_status__in=state_list, document_id=document_id)
page_desc(query_set, 10, generate_problem, is_the_task_interrupted) page_desc(query_set, 10, generate_problem, is_the_task_interrupted)
except Exception as e: except Exception as e:
max_kb_error.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}') maxkb_error_logger.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}')
max_kb_error.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format( maxkb_error_logger.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format(
document_id=document_id, error=str(e), traceback=traceback.format_exc())) document_id=document_id, error=str(e), traceback=traceback.format_exc()))
finally: finally:
ListenerManagement.post_update_document_status(document_id, TaskType.GENERATE_PROBLEM) ListenerManagement.post_update_document_status(document_id, TaskType.GENERATE_PROBLEM)
max_kb.info(_('End--->Generate problem: {document_id}').format(document_id=document_id)) maxkb_logger.info(_('End--->Generate problem: {document_id}').format(document_id=document_id))
@celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']}, @celery_app.task(base=QueueOnce, once={'keys': ['paragraph_id_list']},

6
apps/knowledge/task/handler.py
View File

@ -9,12 +9,10 @@ from django.db.models import QuerySet
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from common.utils.fork import ChildLink, Fork from common.utils.fork import ChildLink, Fork
from common.utils.logger import maxkb_error_logger
from common.utils.split_model import get_split_model from common.utils.split_model import get_split_model
from knowledge.models.knowledge import KnowledgeType, Document, Knowledge, Status from knowledge.models.knowledge import KnowledgeType, Document, Knowledge, Status
max_kb_error = logging.getLogger("max_kb_error")
max_kb = logging.getLogger("max_kb")
def get_save_handler(knowledge_id, selector): def get_save_handler(knowledge_id, selector):
from knowledge.serializers.document import DocumentSerializers from knowledge.serializers.document import DocumentSerializers
@ -118,4 +116,4 @@ def save_problem(knowledge_id, document_id, paragraph_id, problem):
} }
).save(instance={"content": problem}, with_valid=True) ).save(instance={"content": problem}, with_valid=True)
except Exception as e: except Exception as e:
max_kb_error.error(_('Association problem failed {error}').format(error=str(e))) maxkb_error_logger.error(_('Association problem failed {error}').format(error=str(e)))

16
apps/knowledge/task/sync.py
View File

@ -15,10 +15,10 @@ from celery_once import QueueOnce
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from common.utils.fork import ForkManage, Fork from common.utils.fork import ForkManage, Fork
from common.utils.logger import maxkb_logger, maxkb_error_logger
from ops import celery_app from ops import celery_app
max_kb_error = logging.getLogger("max_kb_error")
max_kb = logging.getLogger("max_kb")
@celery_app.task(base=QueueOnce, once={'keys': ['knowledge_id']}, name='celery:sync_web_knowledge') @celery_app.task(base=QueueOnce, once={'keys': ['knowledge_id']}, name='celery:sync_web_knowledge')
@ -26,15 +26,15 @@ def sync_web_knowledge(knowledge_id: str, url: str, selector: str):
from knowledge.task.handler import get_save_handler from knowledge.task.handler import get_save_handler
try: try:
max_kb.info( maxkb_logger.info(
_('Start--->Start synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) _('Start--->Start synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
ForkManage(url, selector.split(" ") if selector is not None else []).fork(2, set(), ForkManage(url, selector.split(" ") if selector is not None else []).fork(2, set(),
get_save_handler(knowledge_id, get_save_handler(knowledge_id,
selector)) selector))
max_kb.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
except Exception as e: except Exception as e:
max_kb_error.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format( maxkb_error_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format(
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc())) knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
@ -43,14 +43,14 @@ def sync_replace_web_knowledge(knowledge_id: str, url: str, selector: str):
from knowledge.task.handler import get_sync_handler from knowledge.task.handler import get_sync_handler
try: try:
max_kb.info( maxkb_logger.info(
_('Start--->Start synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) _('Start--->Start synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
ForkManage(url, selector.split(" ") if selector is not None else []).fork(2, set(), ForkManage(url, selector.split(" ") if selector is not None else []).fork(2, set(),
get_sync_handler(knowledge_id get_sync_handler(knowledge_id
)) ))
max_kb.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id)) maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
except Exception as e: except Exception as e:
max_kb_error.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format( maxkb_error_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format(
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc())) knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))

20
apps/models_provider/impl/volcanic_engine_model_provider/model/tti.py
View File

@ -18,6 +18,7 @@ from typing import Dict
import requests import requests
from common.utils.logger import maxkb_logger
from models_provider.base_model_provider import MaxKBBaseModel from models_provider.base_model_provider import MaxKBBaseModel
from models_provider.impl.base_tti import BaseTextToImage from models_provider.impl.base_tti import BaseTextToImage
@ -35,7 +36,6 @@ req_key_dict = {
'anime_v1.3.1': 'high_aes', 'anime_v1.3.1': 'high_aes',
} }
max_kb = logging.getLogger("max_kb")
def sign(key, msg): def sign(key, msg):
@ -60,7 +60,7 @@ def formatQuery(parameters):
def signV4Request(access_key, secret_key, service, req_query, req_body): def signV4Request(access_key, secret_key, service, req_query, req_body):
if access_key is None or secret_key is None: if access_key is None or secret_key is None:
max_kb.info('No access key is available.') maxkb_logger.info('No access key is available.')
sys.exit() sys.exit()
t = datetime.datetime.utcnow() t = datetime.datetime.utcnow()
@ -77,42 +77,36 @@ def signV4Request(access_key, secret_key, service, req_query, req_body):
'\n' + 'x-date:' + current_date + '\n' '\n' + 'x-date:' + current_date + '\n'
canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + \ canonical_request = method + '\n' + canonical_uri + '\n' + canonical_querystring + \
'\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash '\n' + canonical_headers + '\n' + signed_headers + '\n' + payload_hash
# max_kb.info(canonical_request)
algorithm = 'HMAC-SHA256' algorithm = 'HMAC-SHA256'
credential_scope = datestamp + '/' + region + '/' + service + '/' + 'request' credential_scope = datestamp + '/' + region + '/' + service + '/' + 'request'
string_to_sign = algorithm + '\n' + current_date + '\n' + credential_scope + '\n' + hashlib.sha256( string_to_sign = algorithm + '\n' + current_date + '\n' + credential_scope + '\n' + hashlib.sha256(
canonical_request.encode('utf-8')).hexdigest() canonical_request.encode('utf-8')).hexdigest()
# max_kb.info(string_to_sign)
signing_key = getSignatureKey(secret_key, datestamp, region, service) signing_key = getSignatureKey(secret_key, datestamp, region, service)
# max_kb.info(signing_key)
signature = hmac.new(signing_key, (string_to_sign).encode( signature = hmac.new(signing_key, (string_to_sign).encode(
'utf-8'), hashlib.sha256).hexdigest() 'utf-8'), hashlib.sha256).hexdigest()
# max_kb.info(signature)
authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + \ authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + \
credential_scope + ', ' + 'SignedHeaders=' + \ credential_scope + ', ' + 'SignedHeaders=' + \
signed_headers + ', ' + 'Signature=' + signature signed_headers + ', ' + 'Signature=' + signature
# max_kb.info(authorization_header)
headers = {'X-Date': current_date, headers = {'X-Date': current_date,
'Authorization': authorization_header, 'Authorization': authorization_header,
'X-Content-Sha256': payload_hash, 'X-Content-Sha256': payload_hash,
'Content-Type': content_type 'Content-Type': content_type
} }
# max_kb.info(headers)
# ************* SEND THE REQUEST ************* # ************* SEND THE REQUEST *************
request_url = endpoint + '?' + canonical_querystring request_url = endpoint + '?' + canonical_querystring
max_kb.info('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++') maxkb_logger.info('\nBEGIN REQUEST++++++++++++++++++++++++++++++++++++')
max_kb.info('Request URL = ' + request_url) maxkb_logger.info('Request URL = ' + request_url)
try: try:
r = requests.post(request_url, headers=headers, data=req_body) r = requests.post(request_url, headers=headers, data=req_body)
except Exception as err: except Exception as err:
max_kb.info(f'error occurred: {err}') maxkb_logger.info(f'error occurred: {err}')
raise raise
else: else:
max_kb.info('\nRESPONSE++++++++++++++++++++++++++++++++++++') maxkb_logger.info('\nRESPONSE++++++++++++++++++++++++++++++++++++')
max_kb.info(f'Response code: {r.status_code}\n') maxkb_logger.info(f'Response code: {r.status_code}\n')
# 使用 replace 方法将 \u0026 替换为 & # 使用 replace 方法将 \u0026 替换为 &
resp_str = r.text.replace("\\u0026", "&") resp_str = r.text.replace("\\u0026", "&")
if r.status_code != 200: if r.status_code != 200:

1
apps/models_provider/impl/xf_model_provider/model/stt.py
View File

@ -28,7 +28,6 @@ ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ssl_context.check_hostname = False ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE ssl_context.verify_mode = ssl.CERT_NONE
max_kb = logging.getLogger("max_kb")
class XFSparkSpeechToText(MaxKBBaseModel, BaseSpeechToText): class XFSparkSpeechToText(MaxKBBaseModel, BaseSpeechToText):

1
apps/models_provider/impl/xf_model_provider/model/tts.py
View File

@ -23,7 +23,6 @@ from common.utils.common import _remove_empty_lines
from models_provider.base_model_provider import MaxKBBaseModel from models_provider.base_model_provider import MaxKBBaseModel
from models_provider.impl.base_tts import BaseTextToSpeech from models_provider.impl.base_tts import BaseTextToSpeech
max_kb = logging.getLogger("max_kb")
STATUS_FIRST_FRAME = 0 # 第一帧的标识 STATUS_FIRST_FRAME = 0 # 第一帧的标识
STATUS_CONTINUE_FRAME = 1 # 中间帧标识 STATUS_CONTINUE_FRAME = 1 # 中间帧标识

5
apps/ops/celery/logger.py
View File

@ -8,6 +8,7 @@ from django.conf import settings
from kombu import Connection, Exchange, Queue, Producer from kombu import Connection, Exchange, Queue, Producer
from kombu.mixins import ConsumerMixin from kombu.mixins import ConsumerMixin
from common.utils.logger import maxkb_logger
from .utils import get_celery_task_log_path from .utils import get_celery_task_log_path
from .const import CELERY_LOG_MAGIC_MARK from .const import CELERY_LOG_MAGIC_MARK
@ -209,7 +210,7 @@ class CeleryThreadTaskFileHandler(CeleryThreadingLoggerHandler):
f.flush() f.flush()
def handle_task_start(self, task_id): def handle_task_start(self, task_id):
logging.getLogger("max_kb").info('handle_task_start') maxkb_logger.info('handle_task_start')
log_path = get_celery_task_log_path(task_id) log_path = get_celery_task_log_path(task_id)
thread_id = self.get_current_thread_id() thread_id = self.get_current_thread_id()
self.task_id_thread_id_mapper[task_id] = thread_id self.task_id_thread_id_mapper[task_id] = thread_id
@ -217,7 +218,7 @@ class CeleryThreadTaskFileHandler(CeleryThreadingLoggerHandler):
self.thread_id_fd_mapper[thread_id] = f self.thread_id_fd_mapper[thread_id] = f
def handle_task_end(self, task_id): def handle_task_end(self, task_id):
logging.getLogger('max_kb').info('handle_task_end') maxkb_logger.info('handle_task_end')
ident_id = self.task_id_thread_id_mapper.get(task_id, '') ident_id = self.task_id_thread_id_mapper.get(task_id, '')
f = self.thread_id_fd_mapper.pop(ident_id, None) f = self.thread_id_fd_mapper.pop(ident_id, None)
if f and not f.closed: if f and not f.closed:

3
apps/system_manage/serializers/email_setting.py
View File

@ -15,6 +15,7 @@ from rest_framework import serializers
from common.exception.app_exception import AppApiException from common.exception.app_exception import AppApiException
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from common.utils.logger import maxkb_logger
from system_manage.models import SystemSetting, SettingType from system_manage.models import SystemSetting, SettingType
@ -47,7 +48,7 @@ class EmailSettingSerializer(serializers.Serializer):
self.data.get("email_use_ssl") self.data.get("email_use_ssl")
).open() ).open()
except Exception as e: except Exception as e:
logging.getLogger("max_kb").error(f'Exception: {e}') maxkb_logger.error(f'Exception: {e}')
raise AppApiException(1004, _('Email verification failed')) raise AppApiException(1004, _('Email verification failed'))
def update_or_save(self): def update_or_save(self):