zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修复openai计算token错误

Browse Source
This commit is contained in:
shaohuzhang1 2024-08-29 17:21:19 +08:00 committed by shaohuzhang1
parent b0931c8612
commit 3d1b3ea8d5
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
apps/setting/models_provider/impl/openai_model_provider/model/llm.py
View File

@ -46,14 +46,14 @@ class OpenAIChatModel(MaxKBBaseModel, ChatOpenAI):
def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int: def get_num_tokens_from_messages(self, messages: List[BaseMessage]) -> int:
try: try:
super().get_num_tokens_from_messages(messages) return super().get_num_tokens_from_messages(messages)
except Exception as e: except Exception as e:
tokenizer = TokenizerManage.get_tokenizer() tokenizer = TokenizerManage.get_tokenizer()
return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages]) return sum([len(tokenizer.encode(get_buffer_string([m]))) for m in messages])
def get_num_tokens(self, text: str) -> int: def get_num_tokens(self, text: str) -> int:
try: try:
super().get_num_tokens(text) return super().get_num_tokens(text)
except Exception as e: except Exception as e:
tokenizer = TokenizerManage.get_tokenizer() tokenizer = TokenizerManage.get_tokenizer()
return len(tokenizer.encode(text)) return len(tokenizer.encode(text))