zhuchaowe/maxkb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 同步web站点 选择器不填无法获取内容

Browse Source
This commit is contained in:
shaohuzhang1 2024-01-24 11:23:16 +08:00
parent 7f7ad8b3cd
commit ef66af639e
3 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/common/event/listener_manage.py
View File

@ -181,7 +181,7 @@ class ListenerManagement:
def sync_web_dataset(args: SyncWebDatasetArgs): def sync_web_dataset(args: SyncWebDatasetArgs):
if try_lock('sync_web_dataset' + args.lock_key): if try_lock('sync_web_dataset' + args.lock_key):
try: try:
ForkManage(args.url, args.selector.split(" ")).fork(2, set(), ForkManage(args.url, args.selector.split(" ") if args.selector is not None else []).fork(2, set(),
args.handler) args.handler)
except Exception as e: except Exception as e:
logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}') logging.getLogger("max_kb_error").error(f'{str(e)}:{traceback.format_exc()}')

2
apps/common/util/fork.py
View File

@ -59,7 +59,7 @@ class Fork:
def __init__(self, base_fork_url: str, selector_list: List[str]): def __init__(self, base_fork_url: str, selector_list: List[str]):
self.base_fork_url = urljoin(base_fork_url if base_fork_url.endswith("/") else base_fork_url + '/', '.') self.base_fork_url = urljoin(base_fork_url if base_fork_url.endswith("/") else base_fork_url + '/', '.')
self.base_fork_url = self.base_fork_url[:-1] self.base_fork_url = self.base_fork_url[:-1]
self.selector_list = selector_list self.selector_list = [selector for selector in selector_list if selector is not None and len(selector) > 0]
self.urlparse = urlparse(self.base_fork_url) self.urlparse = urlparse(self.base_fork_url)
self.base_url = ParseResult(scheme=self.urlparse.scheme, netloc=self.urlparse.netloc, path='', params='', self.base_url = ParseResult(scheme=self.urlparse.scheme, netloc=self.urlparse.netloc, path='', params='',
query='', query='',

3
apps/dataset/serializers/document_serializers.py
View File

@ -166,7 +166,8 @@ class DocumentSerializers(ApiMixin, serializers.Serializer):
document.status = Status.embedding document.status = Status.embedding
document.save() document.save()
source_url = document.meta.get('source_url') source_url = document.meta.get('source_url')
selector_list = document.meta.get('selector').split(" ") if 'selector' in document.meta else [] selector_list = document.meta.get('selector').split(
" ") if 'selector' in document.meta and document.meta.get('selector') is not None else []
result = Fork(source_url, selector_list).fork() result = Fork(source_url, selector_list).fork()
if result.status == 200: if result.status == 200:
# 删除段落 # 删除段落