zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: Add custom token_ids method (#2264)

Browse Source
This commit is contained in:
shaohuzhang1 2025-02-13 15:40:09 +08:00 committed by GitHub
parent dac0a9c3f3
commit 051dc13322
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
apps/setting/models_provider/impl/base_chat_open_ai.py
View File

@ -15,8 +15,14 @@ from langchain_openai.chat_models.base import _convert_chunk_to_generation_chunk
from common.config.tokenizer_manage_config import TokenizerManage from common.config.tokenizer_manage_config import TokenizerManage
def custom_get_token_ids(text: str):
tokenizer = TokenizerManage.get_tokenizer()
return tokenizer.encode(text)
class BaseChatOpenAI(ChatOpenAI): class BaseChatOpenAI(ChatOpenAI):
usage_metadata: dict = {} usage_metadata: dict = {}
custom_get_token_ids = custom_get_token_ids
def get_last_generation_info(self) -> Optional[Dict[str, Any]]: def get_last_generation_info(self) -> Optional[Dict[str, Any]]:
return self.usage_metadata return self.usage_metadata