refactor: replace logging with maxkb_logger for consistent logging across modules
This commit is contained in:
parent
6724f6e9d1
commit
a73e0b10f9
@ -26,6 +26,7 @@ from application.chat_pipeline.pipeline_manage import PipelineManage
|
|||||||
from application.chat_pipeline.step.chat_step.i_chat_step import IChatStep, PostResponseHandler
|
from application.chat_pipeline.step.chat_step.i_chat_step import IChatStep, PostResponseHandler
|
||||||
from application.flow.tools import Reasoning
|
from application.flow.tools import Reasoning
|
||||||
from application.models import ApplicationChatUserStats, ChatUserType
|
from application.models import ApplicationChatUserStats, ChatUserType
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from models_provider.tools import get_model_instance_by_model_workspace_id
|
from models_provider.tools import get_model_instance_by_model_workspace_id
|
||||||
|
|
||||||
|
|
||||||
@ -135,7 +136,7 @@ def event_content(response,
|
|||||||
'node_type': 'ai-chat-node'})
|
'node_type': 'ai-chat-node'})
|
||||||
add_access_num(client_id, client_type, manage.context.get('application_id'))
|
add_access_num(client_id, client_type, manage.context.get('application_id'))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
all_text = 'Exception:' + str(e)
|
all_text = 'Exception:' + str(e)
|
||||||
write_context(step, manage, 0, 0, all_text)
|
write_context(step, manage, 0, 0, all_text)
|
||||||
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
|
post_response_handler.handler(chat_id, chat_record_id, paragraph_list, problem_text,
|
||||||
|
|||||||
@ -21,7 +21,7 @@ from common.config.embedding_config import VectorStore
|
|||||||
from common.db.search import native_search, get_dynamics_model, native_update
|
from common.db.search import native_search, get_dynamics_model, native_update
|
||||||
from common.utils.common import get_file_content
|
from common.utils.common import get_file_content
|
||||||
from common.utils.lock import try_lock, un_lock
|
from common.utils.lock import try_lock, un_lock
|
||||||
from common.utils.logger import maxkb_logger, maxkb_error_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from common.utils.page_utils import page_desc
|
from common.utils.page_utils import page_desc
|
||||||
from knowledge.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State,SourceType, SearchMode
|
from knowledge.models import Paragraph, Status, Document, ProblemParagraphMapping, TaskType, State,SourceType, SearchMode
|
||||||
from maxkb.conf import (PROJECT_DIR)
|
from maxkb.conf import (PROJECT_DIR)
|
||||||
@ -86,7 +86,7 @@ class ListenerManagement:
|
|||||||
ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list=paragraph_id_list,
|
ListenerManagement.embedding_by_paragraph_data_list(data_list, paragraph_id_list=paragraph_id_list,
|
||||||
embedding_model=embedding_model)
|
embedding_model=embedding_model)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format(
|
maxkb_logger.error(_('Query vector data: {paragraph_id_list} error {error} {traceback}').format(
|
||||||
paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
|
paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@ -103,7 +103,7 @@ class ListenerManagement:
|
|||||||
# 批量向量化
|
# 批量向量化
|
||||||
VectorStore.get_embedding_vector().batch_save(data_list, embedding_model, is_save_function)
|
VectorStore.get_embedding_vector().batch_save(data_list, embedding_model, is_save_function)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format(
|
maxkb_logger.error(_('Vectorized paragraph: {paragraph_id_list} error {error} {traceback}').format(
|
||||||
paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
|
paragraph_id_list=paragraph_id_list, error=str(e), traceback=traceback.format_exc()))
|
||||||
status = Status.error
|
status = Status.error
|
||||||
finally:
|
finally:
|
||||||
@ -143,7 +143,7 @@ class ListenerManagement:
|
|||||||
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
|
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
|
||||||
State.SUCCESS)
|
State.SUCCESS)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format(
|
maxkb_logger.error(_('Vectorized paragraph: {paragraph_id} error {error} {traceback}').format(
|
||||||
paragraph_id=paragraph_id, error=str(e), traceback=traceback.format_exc()))
|
paragraph_id=paragraph_id, error=str(e), traceback=traceback.format_exc()))
|
||||||
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
|
ListenerManagement.update_status(QuerySet(Paragraph).filter(id=paragraph_id), TaskType.EMBEDDING,
|
||||||
State.FAILURE)
|
State.FAILURE)
|
||||||
@ -284,7 +284,7 @@ class ListenerManagement:
|
|||||||
document_id)),
|
document_id)),
|
||||||
is_the_task_interrupted)
|
is_the_task_interrupted)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Vectorized document: {document_id} error {error} {traceback}').format(
|
maxkb_logger.error(_('Vectorized document: {document_id} error {error} {traceback}').format(
|
||||||
document_id=document_id, error=str(e), traceback=traceback.format_exc()))
|
document_id=document_id, error=str(e), traceback=traceback.format_exc()))
|
||||||
finally:
|
finally:
|
||||||
ListenerManagement.post_update_document_status(document_id, TaskType.EMBEDDING)
|
ListenerManagement.post_update_document_status(document_id, TaskType.EMBEDDING)
|
||||||
@ -308,7 +308,7 @@ class ListenerManagement:
|
|||||||
for document in document_list:
|
for document in document_list:
|
||||||
ListenerManagement.embedding_by_document(document.id, embedding_model=embedding_model)
|
ListenerManagement.embedding_by_document(document.id, embedding_model=embedding_model)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format(
|
maxkb_logger.error(_('Vectorized knowledge: {knowledge_id} error {error} {traceback}').format(
|
||||||
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
|
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
|
||||||
finally:
|
finally:
|
||||||
maxkb_logger.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
|
maxkb_logger.info(_('End--->Embedding knowledge: {knowledge_id}').format(knowledge_id=knowledge_id))
|
||||||
|
|||||||
@ -17,6 +17,8 @@ from common.exception.app_exception import AppApiException
|
|||||||
|
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
|
|
||||||
|
|
||||||
def to_result(key, args, parent_key=None):
|
def to_result(key, args, parent_key=None):
|
||||||
"""
|
"""
|
||||||
@ -96,6 +98,6 @@ def handle_exception(exc, context):
|
|||||||
if issubclass(exception_class, APIException):
|
if issubclass(exception_class, APIException):
|
||||||
return result.error(exc.detail)
|
return result.error(exc.detail)
|
||||||
if response is None:
|
if response is None:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(exc)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(exc)}:{traceback.format_exc()}')
|
||||||
return result.error(str(exc))
|
return result.error(str(exc))
|
||||||
return response
|
return response
|
||||||
|
|||||||
@ -17,6 +17,7 @@ from common.exception.app_exception import AppApiException
|
|||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from common.result import result
|
from common.result import result
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
|
|
||||||
|
|
||||||
def to_result(key, args, parent_key=None):
|
def to_result(key, args, parent_key=None):
|
||||||
@ -89,6 +90,6 @@ def handle_exception(exc, context):
|
|||||||
if issubclass(exception_class, APIException):
|
if issubclass(exception_class, APIException):
|
||||||
return result.error(exc.detail)
|
return result.error(exc.detail)
|
||||||
if response is None:
|
if response is None:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(exc)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(exc)}:{traceback.format_exc()}')
|
||||||
return result.error(str(exc))
|
return result.error(str(exc))
|
||||||
return response
|
return response
|
||||||
|
|||||||
@ -21,6 +21,7 @@ from openpyxl.packaging.relationship import get_rels_path, get_dependents
|
|||||||
from openpyxl.xml.constants import SHEET_DRAWING_NS, REL_NS, SHEET_MAIN_NS
|
from openpyxl.xml.constants import SHEET_DRAWING_NS, REL_NS, SHEET_MAIN_NS
|
||||||
|
|
||||||
from common.handle.base_parse_qa_handle import get_title_row_index_dict, get_row_value
|
from common.handle.base_parse_qa_handle import get_title_row_index_dict, get_row_value
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from knowledge.models import File
|
from knowledge.models import File
|
||||||
|
|
||||||
|
|
||||||
@ -75,7 +76,7 @@ def handle_images(deps, archive: ZipFile) -> []:
|
|||||||
image_io = archive.read(dep.target)
|
image_io = archive.read(dep.target)
|
||||||
image = openpyxl_Image(BytesIO(image_io))
|
image = openpyxl_Image(BytesIO(image_io))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f"Error reading image {dep.target}: {e}")
|
maxkb_logger.error(f"Error reading image {dep.target}: {e}")
|
||||||
continue
|
continue
|
||||||
image.embed = dep.id # 文件rId
|
image.embed = dep.id # 文件rId
|
||||||
image.target = dep.target # 文件地址
|
image.target = dep.target # 文件地址
|
||||||
|
|||||||
@ -4,7 +4,7 @@ import logging
|
|||||||
from charset_normalizer import detect
|
from charset_normalizer import detect
|
||||||
|
|
||||||
from common.handle.base_parse_table_handle import BaseParseTableHandle
|
from common.handle.base_parse_table_handle import BaseParseTableHandle
|
||||||
from common.utils.logger import maxkb_logger, maxkb_error_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
|
||||||
|
|
||||||
class CsvParseTableHandle(BaseParseTableHandle):
|
class CsvParseTableHandle(BaseParseTableHandle):
|
||||||
@ -39,5 +39,5 @@ class CsvParseTableHandle(BaseParseTableHandle):
|
|||||||
try:
|
try:
|
||||||
return buffer.decode(detect(buffer)['encoding'])
|
return buffer.decode(detect(buffer)['encoding'])
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_error_logger.error(f'csv split handle error: {e}')
|
maxkb_logger.error(f'csv split handle error: {e}')
|
||||||
return f'error: {e}'
|
return f'error: {e}'
|
||||||
|
|||||||
@ -4,7 +4,7 @@ import logging
|
|||||||
import xlrd
|
import xlrd
|
||||||
|
|
||||||
from common.handle.base_parse_table_handle import BaseParseTableHandle
|
from common.handle.base_parse_table_handle import BaseParseTableHandle
|
||||||
from common.utils.logger import maxkb_error_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
|
||||||
|
|
||||||
class XlsParseTableHandle(BaseParseTableHandle):
|
class XlsParseTableHandle(BaseParseTableHandle):
|
||||||
@ -55,7 +55,7 @@ class XlsParseTableHandle(BaseParseTableHandle):
|
|||||||
result.append({'name': sheet.name, 'paragraphs': paragraphs})
|
result.append({'name': sheet.name, 'paragraphs': paragraphs})
|
||||||
|
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_error_logger.error(f'excel split handle error: {e}')
|
maxkb_logger.error(f'excel split handle error: {e}')
|
||||||
return [{'name': file.name, 'paragraphs': []}]
|
return [{'name': file.name, 'paragraphs': []}]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@ -6,7 +6,7 @@ from openpyxl import load_workbook
|
|||||||
|
|
||||||
from common.handle.base_parse_table_handle import BaseParseTableHandle
|
from common.handle.base_parse_table_handle import BaseParseTableHandle
|
||||||
from common.handle.impl.common_handle import xlsx_embed_cells_images
|
from common.handle.impl.common_handle import xlsx_embed_cells_images
|
||||||
from common.utils.logger import maxkb_error_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
|
||||||
|
|
||||||
class XlsxParseTableHandle(BaseParseTableHandle):
|
class XlsxParseTableHandle(BaseParseTableHandle):
|
||||||
@ -73,7 +73,7 @@ class XlsxParseTableHandle(BaseParseTableHandle):
|
|||||||
result.append({'name': sheetname, 'paragraphs': paragraphs})
|
result.append({'name': sheetname, 'paragraphs': paragraphs})
|
||||||
|
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_error_logger.error(f'excel split handle error: {e}')
|
maxkb_logger.error(f'excel split handle error: {e}')
|
||||||
return [{'name': file.name, 'paragraphs': []}]
|
return [{'name': file.name, 'paragraphs': []}]
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@ -86,7 +86,7 @@ class XlsxParseTableHandle(BaseParseTableHandle):
|
|||||||
if len(image_dict) > 0:
|
if len(image_dict) > 0:
|
||||||
save_image(image_dict.values())
|
save_image(image_dict.values())
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(f'Exception: {e}')
|
maxkb_logger.error(f'Exception: {e}')
|
||||||
image_dict = {}
|
image_dict = {}
|
||||||
md_tables = ''
|
md_tables = ''
|
||||||
# 如果未指定 sheet_name,则使用第一个工作表
|
# 如果未指定 sheet_name,则使用第一个工作表
|
||||||
|
|||||||
@ -21,6 +21,7 @@ from docx.table import Table
|
|||||||
from docx.text.paragraph import Paragraph
|
from docx.text.paragraph import Paragraph
|
||||||
|
|
||||||
from common.handle.base_split_handle import BaseSplitHandle
|
from common.handle.base_split_handle import BaseSplitHandle
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from common.utils.split_model import SplitModel
|
from common.utils.split_model import SplitModel
|
||||||
from knowledge.models import File
|
from knowledge.models import File
|
||||||
|
|
||||||
@ -79,7 +80,7 @@ def get_paragraph_element_txt(paragraph_element, doc: Document, images_list, get
|
|||||||
return paragraph_element.text
|
return paragraph_element.text
|
||||||
return ""
|
return ""
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'Error getting paragraph element text: {e}')
|
maxkb_logger.error(f'Error getting paragraph element text: {e}')
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -19,7 +19,7 @@ from django.utils.translation import gettext_lazy as _
|
|||||||
from langchain_community.document_loaders import PyPDFLoader
|
from langchain_community.document_loaders import PyPDFLoader
|
||||||
|
|
||||||
from common.handle.base_split_handle import BaseSplitHandle
|
from common.handle.base_split_handle import BaseSplitHandle
|
||||||
from common.utils.logger import maxkb_error_logger, maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from common.utils.split_model import SplitModel
|
from common.utils.split_model import SplitModel
|
||||||
|
|
||||||
default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'),
|
default_pattern_list = [re.compile('(?<=^)# .*|(?<=\\n)# .*'),
|
||||||
@ -72,7 +72,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
|||||||
else:
|
else:
|
||||||
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
split_model = SplitModel(default_pattern_list, with_filter=with_filter, limit=limit)
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_error_logger.error(f"File: {file.name}, error: {e}")
|
maxkb_logger.error(f"File: {file.name}, error: {e}")
|
||||||
return {'name': file.name,
|
return {'name': file.name,
|
||||||
'content': []}
|
'content': []}
|
||||||
finally:
|
finally:
|
||||||
@ -109,7 +109,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
|||||||
raise e
|
raise e
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
# 当页出错继续进行下一页,防止一个页面出错导致整个文件解析失败
|
# 当页出错继续进行下一页,防止一个页面出错导致整个文件解析失败
|
||||||
maxkb_error_logger.error(f"File: {file.name}, Page: {page_num + 1}, error: {e}")
|
maxkb_logger.error(f"File: {file.name}, Page: {page_num + 1}, error: {e}")
|
||||||
continue
|
continue
|
||||||
finally:
|
finally:
|
||||||
os.remove(page_num_pdf)
|
os.remove(page_num_pdf)
|
||||||
@ -278,7 +278,7 @@ class PdfSplitHandle(BaseSplitHandle):
|
|||||||
pre_toc[i]['content'] = re.sub(r'(?<!。)\n+', '', pre_toc[i]['content'])
|
pre_toc[i]['content'] = re.sub(r'(?<!。)\n+', '', pre_toc[i]['content'])
|
||||||
pre_toc[i]['content'] = re.sub(r'(?<!.)\n+', '', pre_toc[i]['content'])
|
pre_toc[i]['content'] = re.sub(r'(?<!.)\n+', '', pre_toc[i]['content'])
|
||||||
except BaseException as e:
|
except BaseException as e:
|
||||||
maxkb_error_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
|
maxkb_logger.error(_('This document has no preface and is treated as ordinary text: {e}').format(e=e))
|
||||||
if pattern_list is not None and len(pattern_list) > 0:
|
if pattern_list is not None and len(pattern_list) > 0:
|
||||||
split_model = SplitModel(pattern_list, with_filter, limit)
|
split_model = SplitModel(pattern_list, with_filter, limit)
|
||||||
else:
|
else:
|
||||||
|
|||||||
@ -10,7 +10,7 @@ import html2text as ht
|
|||||||
import requests
|
import requests
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from common.utils.logger import maxkb_error_logger, maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
|
|
||||||
requests.packages.urllib3.disable_warnings()
|
requests.packages.urllib3.disable_warnings()
|
||||||
|
|
||||||
@ -147,7 +147,7 @@ class Fork:
|
|||||||
try:
|
try:
|
||||||
html_content = response.content.decode(charset)
|
html_content = response.content.decode(charset)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(f'{e}')
|
maxkb_logger.error(f'{e}')
|
||||||
return BeautifulSoup(html_content, "html.parser")
|
return BeautifulSoup(html_content, "html.parser")
|
||||||
return beautiful_soup
|
return beautiful_soup
|
||||||
|
|
||||||
@ -161,11 +161,11 @@ class Fork:
|
|||||||
maxkb_logger.info(f'fork:{self.base_fork_url}')
|
maxkb_logger.info(f'fork:{self.base_fork_url}')
|
||||||
response = requests.get(self.base_fork_url, verify=False, headers=headers)
|
response = requests.get(self.base_fork_url, verify=False, headers=headers)
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
maxkb_error_logger.error(f"url: {self.base_fork_url} code:{response.status_code}")
|
maxkb_logger.error(f"url: {self.base_fork_url} code:{response.status_code}")
|
||||||
return Fork.Response.error(f"url: {self.base_fork_url} code:{response.status_code}")
|
return Fork.Response.error(f"url: {self.base_fork_url} code:{response.status_code}")
|
||||||
bf = self.get_beautiful_soup(response)
|
bf = self.get_beautiful_soup(response)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
return Fork.Response.error(str(e))
|
return Fork.Response.error(str(e))
|
||||||
bf = self.reset_beautiful_soup(bf)
|
bf = self.reset_beautiful_soup(bf)
|
||||||
link_list = self.get_child_link_list(bf)
|
link_list = self.get_child_link_list(bf)
|
||||||
|
|||||||
3
apps/common/utils/logger.py
Normal file
3
apps/common/utils/logger.py
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
import logging
|
||||||
|
|
||||||
|
maxkb_logger = logging.getLogger('max_kb')
|
||||||
@ -42,6 +42,7 @@ from common.handle.impl.text.xlsx_split_handle import XlsxSplitHandle
|
|||||||
from common.handle.impl.text.zip_split_handle import ZipSplitHandle
|
from common.handle.impl.text.zip_split_handle import ZipSplitHandle
|
||||||
from common.utils.common import post, get_file_content, bulk_create_in_batches, parse_image
|
from common.utils.common import post, get_file_content, bulk_create_in_batches, parse_image
|
||||||
from common.utils.fork import Fork
|
from common.utils.fork import Fork
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from common.utils.split_model import get_split_model, flat_map
|
from common.utils.split_model import get_split_model, flat_map
|
||||||
from knowledge.models import Knowledge, Paragraph, Problem, Document, KnowledgeType, ProblemParagraphMapping, State, \
|
from knowledge.models import Knowledge, Paragraph, Problem, Document, KnowledgeType, ProblemParagraphMapping, State, \
|
||||||
TaskType, File, FileSourceType
|
TaskType, File, FileSourceType
|
||||||
@ -483,7 +484,7 @@ class DocumentSerializers(serializers.Serializer):
|
|||||||
else:
|
else:
|
||||||
state = State.FAILURE
|
state = State.FAILURE
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
state = State.FAILURE
|
state = State.FAILURE
|
||||||
ListenerManagement.update_status(
|
ListenerManagement.update_status(
|
||||||
QuerySet(Document).filter(id=document_id),
|
QuerySet(Document).filter(id=document_id),
|
||||||
|
|||||||
@ -647,7 +647,7 @@ class KnowledgeSerializer(serializers.Serializer):
|
|||||||
'selector': knowledge.meta.get('selector')},
|
'selector': knowledge.meta.get('selector')},
|
||||||
'type': KnowledgeType.WEB}, with_valid=True)
|
'type': KnowledgeType.WEB}, with_valid=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
|
|
||||||
return handler
|
return handler
|
||||||
|
|
||||||
|
|||||||
@ -11,7 +11,7 @@ from django.utils.translation import gettext_lazy as _
|
|||||||
from common.config.embedding_config import ModelManage
|
from common.config.embedding_config import ModelManage
|
||||||
from common.event import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \
|
from common.event import ListenerManagement, UpdateProblemArgs, UpdateEmbeddingKnowledgeIdArgs, \
|
||||||
UpdateEmbeddingDocumentIdArgs
|
UpdateEmbeddingDocumentIdArgs
|
||||||
from common.utils.logger import maxkb_error_logger, maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from knowledge.models import Document, TaskType, State
|
from knowledge.models import Document, TaskType, State
|
||||||
from models_provider.tools import get_model
|
from models_provider.tools import get_model
|
||||||
from models_provider.models import Model
|
from models_provider.models import Model
|
||||||
@ -19,7 +19,7 @@ from ops import celery_app
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
def get_embedding_model(model_id, exception_handler=lambda e: maxkb_error_logger.error(
|
def get_embedding_model(model_id, exception_handler=lambda e: maxkb_logger.error(
|
||||||
_('Failed to obtain vector model: {error} {traceback}').format(
|
_('Failed to obtain vector model: {error} {traceback}').format(
|
||||||
error=str(e),
|
error=str(e),
|
||||||
traceback=traceback.format_exc()
|
traceback=traceback.format_exc()
|
||||||
@ -69,7 +69,7 @@ def embedding_by_document(document_id, model_id, state_list=None):
|
|||||||
def exception_handler(e):
|
def exception_handler(e):
|
||||||
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
|
ListenerManagement.update_status(QuerySet(Document).filter(id=document_id), TaskType.EMBEDDING,
|
||||||
State.FAILURE)
|
State.FAILURE)
|
||||||
maxkb_error_logger.error(
|
maxkb_logger.error(
|
||||||
_('Failed to obtain vector model: {error} {traceback}').format(
|
_('Failed to obtain vector model: {error} {traceback}').format(
|
||||||
error=str(e),
|
error=str(e),
|
||||||
traceback=traceback.format_exc()
|
traceback=traceback.format_exc()
|
||||||
@ -111,7 +111,7 @@ def embedding_by_knowledge(knowledge_id, model_id):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
pass
|
pass
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(
|
maxkb_logger.error(
|
||||||
_('Vectorized knowledge: {knowledge_id} error {error} {traceback}'.format(knowledge_id=knowledge_id,
|
_('Vectorized knowledge: {knowledge_id} error {error} {traceback}'.format(knowledge_id=knowledge_id,
|
||||||
error=str(e),
|
error=str(e),
|
||||||
traceback=traceback.format_exc())))
|
traceback=traceback.format_exc())))
|
||||||
|
|||||||
@ -9,7 +9,7 @@ from langchain_core.messages import HumanMessage
|
|||||||
|
|
||||||
from common.config.embedding_config import ModelManage
|
from common.config.embedding_config import ModelManage
|
||||||
from common.event import ListenerManagement
|
from common.event import ListenerManagement
|
||||||
from common.utils.logger import maxkb_error_logger, maxkb_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from common.utils.page_utils import page, page_desc
|
from common.utils.page_utils import page, page_desc
|
||||||
from knowledge.models import Paragraph, Document, Status, TaskType, State
|
from knowledge.models import Paragraph, Document, Status, TaskType, State
|
||||||
from knowledge.task.handler import save_problem
|
from knowledge.task.handler import save_problem
|
||||||
@ -100,8 +100,8 @@ def generate_related_by_document_id(document_id, model_id, prompt, state_list=No
|
|||||||
).filter(task_type_status__in=state_list, document_id=document_id)
|
).filter(task_type_status__in=state_list, document_id=document_id)
|
||||||
page_desc(query_set, 10, generate_problem, is_the_task_interrupted)
|
page_desc(query_set, 10, generate_problem, is_the_task_interrupted)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}')
|
maxkb_logger.error(f'根据文档生成问题:{document_id}出现错误{str(e)}{traceback.format_exc()}')
|
||||||
maxkb_error_logger.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format(
|
maxkb_logger.error(_('Generate issue based on document: {document_id} error {error}{traceback}').format(
|
||||||
document_id=document_id, error=str(e), traceback=traceback.format_exc()))
|
document_id=document_id, error=str(e), traceback=traceback.format_exc()))
|
||||||
finally:
|
finally:
|
||||||
ListenerManagement.post_update_document_status(document_id, TaskType.GENERATE_PROBLEM)
|
ListenerManagement.post_update_document_status(document_id, TaskType.GENERATE_PROBLEM)
|
||||||
|
|||||||
@ -9,7 +9,7 @@ from django.db.models import QuerySet
|
|||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from common.utils.fork import ChildLink, Fork
|
from common.utils.fork import ChildLink, Fork
|
||||||
from common.utils.logger import maxkb_error_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from common.utils.split_model import get_split_model
|
from common.utils.split_model import get_split_model
|
||||||
from knowledge.models.knowledge import KnowledgeType, Document, Knowledge, Status
|
from knowledge.models.knowledge import KnowledgeType, Document, Knowledge, Status
|
||||||
|
|
||||||
@ -32,7 +32,7 @@ def get_save_handler(knowledge_id, selector):
|
|||||||
'type': KnowledgeType.WEB
|
'type': KnowledgeType.WEB
|
||||||
}, with_valid=True)
|
}, with_valid=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
|
|
||||||
return handler
|
return handler
|
||||||
|
|
||||||
@ -64,7 +64,7 @@ def get_sync_handler(knowledge_id):
|
|||||||
'type': KnowledgeType.WEB
|
'type': KnowledgeType.WEB
|
||||||
}, with_valid=True)
|
}, with_valid=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
|
|
||||||
return handler
|
return handler
|
||||||
|
|
||||||
@ -82,7 +82,7 @@ def get_sync_web_document_handler(knowledge_id):
|
|||||||
'meta': {'source_url': source_url, 'selector': selector},
|
'meta': {'source_url': source_url, 'selector': selector},
|
||||||
'type': KnowledgeType.WEB}, with_valid=True)
|
'type': KnowledgeType.WEB}, with_valid=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')
|
maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}')
|
||||||
else:
|
else:
|
||||||
Document(name=source_url[0:128],
|
Document(name=source_url[0:128],
|
||||||
knowledge_id=knowledge_id,
|
knowledge_id=knowledge_id,
|
||||||
@ -116,4 +116,4 @@ def save_problem(knowledge_id, document_id, paragraph_id, problem):
|
|||||||
}
|
}
|
||||||
).save(instance={"content": problem}, with_valid=True)
|
).save(instance={"content": problem}, with_valid=True)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Association problem failed {error}').format(error=str(e)))
|
maxkb_logger.error(_('Association problem failed {error}').format(error=str(e)))
|
||||||
|
|||||||
@ -15,7 +15,7 @@ from celery_once import QueueOnce
|
|||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from common.utils.fork import ForkManage, Fork
|
from common.utils.fork import ForkManage, Fork
|
||||||
from common.utils.logger import maxkb_logger, maxkb_error_logger
|
from common.utils.logger import maxkb_logger
|
||||||
from ops import celery_app
|
from ops import celery_app
|
||||||
|
|
||||||
|
|
||||||
@ -34,7 +34,7 @@ def sync_web_knowledge(knowledge_id: str, url: str, selector: str):
|
|||||||
|
|
||||||
maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
|
maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format(
|
maxkb_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format(
|
||||||
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
|
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
|
||||||
|
|
||||||
|
|
||||||
@ -50,7 +50,7 @@ def sync_replace_web_knowledge(knowledge_id: str, url: str, selector: str):
|
|||||||
))
|
))
|
||||||
maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
|
maxkb_logger.info(_('End--->End synchronization web knowledge base:{knowledge_id}').format(knowledge_id=knowledge_id))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
maxkb_error_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format(
|
maxkb_logger.error(_('Synchronize web knowledge base:{knowledge_id} error{error}{traceback}').format(
|
||||||
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
|
knowledge_id=knowledge_id, error=str(e), traceback=traceback.format_exc()))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@ -7,10 +7,11 @@ from django.utils.translation import gettext
|
|||||||
from langchain_community.chat_models import ChatTongyi
|
from langchain_community.chat_models import ChatTongyi
|
||||||
from langchain_core.messages import HumanMessage
|
from langchain_core.messages import HumanMessage
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from models_provider.base_model_provider import MaxKBBaseModel
|
from models_provider.base_model_provider import MaxKBBaseModel
|
||||||
from models_provider.impl.base_tti import BaseTextToImage
|
from models_provider.impl.base_tti import BaseTextToImage
|
||||||
|
|
||||||
max_kb_error = logging.getLogger("max_kb_error")
|
|
||||||
|
|
||||||
|
|
||||||
class QwenTextToImageModel(MaxKBBaseModel, BaseTextToImage):
|
class QwenTextToImageModel(MaxKBBaseModel, BaseTextToImage):
|
||||||
@ -56,6 +57,6 @@ class QwenTextToImageModel(MaxKBBaseModel, BaseTextToImage):
|
|||||||
for result in rsp.output.results:
|
for result in rsp.output.results:
|
||||||
file_urls.append(result.url)
|
file_urls.append(result.url)
|
||||||
else:
|
else:
|
||||||
max_kb_error.error('sync_call Failed, status_code: %s, code: %s, message: %s' %
|
maxkb_logger.error('sync_call Failed, status_code: %s, code: %s, message: %s' %
|
||||||
(rsp.status_code, rsp.code, rsp.message))
|
(rsp.status_code, rsp.code, rsp.message))
|
||||||
return file_urls
|
return file_urls
|
||||||
|
|||||||
@ -11,10 +11,10 @@ from tencentcloud.common.profile.client_profile import ClientProfile
|
|||||||
from tencentcloud.common.profile.http_profile import HttpProfile
|
from tencentcloud.common.profile.http_profile import HttpProfile
|
||||||
from tencentcloud.hunyuan.v20230901 import hunyuan_client, models
|
from tencentcloud.hunyuan.v20230901 import hunyuan_client, models
|
||||||
|
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from models_provider.base_model_provider import MaxKBBaseModel
|
from models_provider.base_model_provider import MaxKBBaseModel
|
||||||
from models_provider.impl.base_tti import BaseTextToImage
|
from models_provider.impl.base_tti import BaseTextToImage
|
||||||
from models_provider.impl.tencent_model_provider.model.hunyuan import ChatHunyuan
|
from models_provider.impl.tencent_model_provider.model.hunyuan import ChatHunyuan
|
||||||
max_kb_error = logging.getLogger("max_kb_error")
|
|
||||||
|
|
||||||
|
|
||||||
class TencentTextToImageModel(MaxKBBaseModel, BaseTextToImage):
|
class TencentTextToImageModel(MaxKBBaseModel, BaseTextToImage):
|
||||||
@ -89,4 +89,4 @@ class TencentTextToImageModel(MaxKBBaseModel, BaseTextToImage):
|
|||||||
file_urls.append(resp.ResultImage)
|
file_urls.append(resp.ResultImage)
|
||||||
return file_urls
|
return file_urls
|
||||||
except TencentCloudSDKException as err:
|
except TencentCloudSDKException as err:
|
||||||
max_kb_error.error(f"Tencent Text to Image API call failed: {err}")
|
maxkb_logger.error(f"Tencent Text to Image API call failed: {err}")
|
||||||
|
|||||||
@ -23,9 +23,9 @@ from urllib.parse import urlparse
|
|||||||
|
|
||||||
import websockets
|
import websockets
|
||||||
|
|
||||||
|
from common.utils.logger import maxkb_logger
|
||||||
from models_provider.base_model_provider import MaxKBBaseModel
|
from models_provider.base_model_provider import MaxKBBaseModel
|
||||||
from models_provider.impl.base_stt import BaseSpeechToText
|
from models_provider.impl.base_stt import BaseSpeechToText
|
||||||
max_kb_error = logging.getLogger("max_kb_error")
|
|
||||||
|
|
||||||
audio_format = "mp3" # wav 或者 mp3,根据实际音频格式设置
|
audio_format = "mp3" # wav 或者 mp3,根据实际音频格式设置
|
||||||
|
|
||||||
@ -147,7 +147,7 @@ def parse_response(res):
|
|||||||
result['code'] = code
|
result['code'] = code
|
||||||
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
payload_size = int.from_bytes(payload[4:8], "big", signed=False)
|
||||||
payload_msg = payload[8:]
|
payload_msg = payload[8:]
|
||||||
max_kb_error.error(f"Error code: {code}, message: {payload_msg}")
|
maxkb_logger.error(f"Error code: {code}, message: {payload_msg}")
|
||||||
if payload_msg is None:
|
if payload_msg is None:
|
||||||
return result
|
return result
|
||||||
if message_compression == GZIP:
|
if message_compression == GZIP:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user