zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 修改下载模型脚本

Browse Source
This commit is contained in:
zhangshaohu 2024-01-30 19:01:26 +08:00
parent 37376dc26f
commit c29cd7ac24
1 changed files with 6 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
install_model.py
View File

@ -8,11 +8,10 @@
""" """
import json import json
import os.path import os.path
from pycrawlers import huggingface
from transformers import GPT2TokenizerFast from transformers import GPT2TokenizerFast
import sentence_transformers hg = huggingface()
prefix_dir = "./model"
prefix_dir = "/opt/maxkb/model"
model_config = [ model_config = [
{ {
'download_params': { 'download_params': {
@ -51,10 +50,10 @@ model_config = [
}, },
{ {
'download_params': { 'download_params': {
'model_name_or_path': 'shibing624/text2vec-base-chinese', 'urls': ["https://huggingface.co/shibing624/text2vec-base-chinese/tree/main"],
'cache_folder': os.path.join(prefix_dir, 'embedding') 'file_save_paths': [os.path.join(prefix_dir, 'embedding',"shibing624_text2vec-base-chinese")]
}, },
'download_function': sentence_transformers.SentenceTransformer 'download_function': hg.get_batch_data
} }
] ]