Merge branch 'feature/rag_retrive_top_k' into onprem-release
This commit is contained in:
朱潮
commit
7cb1043ae3
@ -88,9 +88,8 @@ skill-name/
|
||||
|
||||
## Skill 加载优先级
|
||||
|
||||
1. Skill MCP 配置（最高）
|
||||
2. 默认 MCP 配置 (`mcp/mcp_settings.json`)
|
||||
3. 用户传入参数（覆盖所有）
|
||||
1. Skill MCP 配置
|
||||
2. 用户传入参数（覆盖已有同名配置）
|
||||
|
||||
## 安全措施
|
||||
|
||||
|
||||
@ -396,7 +396,6 @@ dataset_name/
|
||||
│ ├── document.txt # 原始文本内容
|
||||
│ ├── serialization.txt # 结构化数据
|
||||
│ └── schema.json # 字段定义和元数据
|
||||
├── mcp_settings.json # MCP 工具配置
|
||||
└── system_prompt.md # 系统提示词（可选）
|
||||
```
|
||||
|
||||
@ -405,7 +404,6 @@ dataset_name/
|
||||
- **document.txt**: 原始 Markdown 文本，提供完整上下文
|
||||
- **serialization.txt**: 格式化结构数据，每行 `字段1:值1;字段2:值2`
|
||||
- **schema.json**: 字段定义、枚举值映射和文件关联关系
|
||||
- **mcp_settings.json**: MCP 工具配置，定义可用的数据处理工具
|
||||
|
||||
---
|
||||
|
||||
@ -565,8 +563,7 @@ qwen-agent/
|
||||
│ ├── multi_keyword_search_server.py # 多关键词搜索服务
|
||||
│ ├── excel_csv_operator_server.py # Excel/CSV 操作服务
|
||||
│ ├── json_reader_server.py # JSON 读取服务
|
||||
│ ├── mcp_settings.json # MCP 配置文件
|
||||
│ └── tools/ # 工具定义文件
|
||||
│ └── tools/ # 工具定义文件
|
||||
├── models/ # 模型文件
|
||||
├── projects/ # 项目目录
|
||||
│ └── queue_data/ # 队列数据
|
||||
|
||||
@ -117,13 +117,6 @@ def read_system_prompt():
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def read_mcp_settings():
|
||||
"""读取MCP工具配置"""
|
||||
with open("./mcp/mcp_settings.json", "r") as f:
|
||||
mcp_settings_json = json.load(f)
|
||||
return mcp_settings_json
|
||||
|
||||
|
||||
async def get_tools_from_mcp(mcp):
|
||||
"""从MCP配置中提取工具（带缓存）"""
|
||||
start_time = time.time()
|
||||
@ -195,8 +188,7 @@ async def init_agent(config: AgentConfig):
|
||||
final_system_prompt = await load_system_prompt_async(config)
|
||||
final_mcp_settings = await load_mcp_settings_async(config)
|
||||
|
||||
# 如果没有提供mcp，使用config中的mcp_settings
|
||||
mcp_settings = final_mcp_settings if final_mcp_settings else read_mcp_settings()
|
||||
mcp_settings = final_mcp_settings if final_mcp_settings else []
|
||||
system_prompt = final_system_prompt if final_system_prompt else read_system_prompt()
|
||||
|
||||
config.system_prompt = mcp_settings
|
||||
|
||||
@ -5,6 +5,7 @@ Claude Plugins 模式的 Hook 加载器
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import copy
|
||||
import logging
|
||||
import asyncio
|
||||
import subprocess
|
||||
@ -116,7 +117,8 @@ async def merge_skill_mcp_configs(bot_id: str) -> List[Dict]:
|
||||
plugin_config = json.load(f)
|
||||
servers = plugin_config.get('mcpServers', {})
|
||||
if servers:
|
||||
merged_servers.update(servers)
|
||||
normalized_servers = _normalize_skill_mcp_servers(servers, skill_path)
|
||||
merged_servers.update(normalized_servers)
|
||||
logger.info(f"Loaded MCP config from skill: {skill_name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load mcpServers from {skill_name}: {e}")
|
||||
@ -127,6 +129,47 @@ async def merge_skill_mcp_configs(bot_id: str) -> List[Dict]:
|
||||
return []
|
||||
|
||||
|
||||
def _normalize_skill_mcp_servers(servers: Dict[str, Any], skill_path: str) -> Dict[str, Any]:
|
||||
"""将 skill plugin 中 stdio MCP server 的相对路径归一化为基于 skill 目录的绝对路径。"""
|
||||
normalized_servers = copy.deepcopy(servers)
|
||||
|
||||
for server_name, server_config in normalized_servers.items():
|
||||
if not isinstance(server_config, dict):
|
||||
continue
|
||||
|
||||
transport = server_config.get('transport')
|
||||
if not transport:
|
||||
transport = 'http' if 'url' in server_config else 'stdio'
|
||||
if transport != 'stdio':
|
||||
continue
|
||||
|
||||
command = server_config.get('command')
|
||||
if isinstance(command, str):
|
||||
server_config['command'] = _resolve_skill_relative_path(command, skill_path)
|
||||
|
||||
args = server_config.get('args')
|
||||
if isinstance(args, list):
|
||||
server_config['args'] = [
|
||||
_resolve_skill_relative_path(arg, skill_path) if isinstance(arg, str) else arg
|
||||
for arg in args
|
||||
]
|
||||
|
||||
return normalized_servers
|
||||
|
||||
|
||||
def _resolve_skill_relative_path(value: str, skill_path: str) -> str:
|
||||
"""将 ./ 或 ../ 开头且不含占位符的路径转为基于 skill 目录的绝对路径。"""
|
||||
if '{' in value or '}' in value:
|
||||
return value
|
||||
|
||||
if not value.startswith(('./', '../')):
|
||||
return value
|
||||
|
||||
normalized_path = os.path.abspath(os.path.join(skill_path, value))
|
||||
logger.debug(f"Resolved skill MCP path: {value} -> {normalized_path}")
|
||||
return normalized_path
|
||||
|
||||
|
||||
def _load_plugin_config(plugin_json_path: str) -> Dict:
|
||||
"""加载 plugin.json 配置"""
|
||||
try:
|
||||
|
||||
@ -203,10 +203,9 @@ async def load_mcp_settings_async(config) -> List[Dict]:
|
||||
List[Dict]: 合并后的MCP设置列表
|
||||
|
||||
Note:
|
||||
支持在 mcp_settings.json 的 args 中使用 {dataset_dir} 占位符，
|
||||
支持在传入或合并后的 mcp_settings 的 args 中使用 {dataset_dir} 占位符，
|
||||
会在 init_modified_agent_service_with_files 中被替换为实际的路径。
|
||||
"""
|
||||
from agent.config_cache import config_cache
|
||||
|
||||
# 从config中获取参数
|
||||
project_dir = getattr(config, 'project_dir', None)
|
||||
@ -222,33 +221,6 @@ async def load_mcp_settings_async(config) -> List[Dict]:
|
||||
skill_mcp_servers = skill_mcp_settings[0].get('mcpServers', {})
|
||||
logger.info(f"Loaded {len(skill_mcp_servers)} MCP servers from skills")
|
||||
# ===========================================================================================
|
||||
|
||||
# 2. 读取默认MCP设置（使用缓存）
|
||||
default_mcp_settings = []
|
||||
try:
|
||||
default_mcp_file = os.path.join("mcp", f"mcp_settings.json")
|
||||
default_mcp_settings = await config_cache.get_json_file(default_mcp_file) or []
|
||||
if default_mcp_settings:
|
||||
logger.info(f"Using cached default mcp_settings from mcp folder")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load default mcp_settings: {str(e)}")
|
||||
default_mcp_settings = []
|
||||
|
||||
# 3. 合并默认设置到merged_settings（默认设置被skill覆盖）
|
||||
if default_mcp_settings and len(default_mcp_settings) > 0:
|
||||
default_mcp_servers = default_mcp_settings[0].get('mcpServers', {})
|
||||
if merged_settings and len(merged_settings) > 0:
|
||||
# skill配置已存在，将默认配置合并进去（skill优先）
|
||||
skill_mcp_servers = merged_settings[0].get('mcpServers', {})
|
||||
# 默认配置中不存在的才添加
|
||||
for server_name, server_config in default_mcp_servers.items():
|
||||
if server_name not in skill_mcp_servers:
|
||||
skill_mcp_servers[server_name] = server_config
|
||||
else:
|
||||
# 没有skill配置，直接使用默认配置
|
||||
merged_settings = default_mcp_settings.copy()
|
||||
|
||||
# 遍历mcpServers工具，给每个工具增加env参数
|
||||
if merged_settings and len(merged_settings) > 0:
|
||||
mcp_servers = merged_settings[0].get('mcpServers', {})
|
||||
for server_name, server_config in mcp_servers.items():
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
[
|
||||
{
|
||||
"mcpServers": {}
|
||||
}
|
||||
]
|
||||
@ -14,7 +14,7 @@
|
||||
"transport": "stdio",
|
||||
"command": "python",
|
||||
"args": [
|
||||
"./skills/rag-retrieve-only/rag_retrieve_server.py",
|
||||
"./rag_retrieve_server.py",
|
||||
"{bot_id}"
|
||||
]
|
||||
}
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
"transport": "stdio",
|
||||
"command": "python",
|
||||
"args": [
|
||||
"./skills_autoload/rag-retrieve/rag_retrieve_server.py",
|
||||
"./rag_retrieve_server.py",
|
||||
"{bot_id}"
|
||||
]
|
||||
}
|
||||
|
||||
@ -401,23 +401,30 @@ def _extract_skills_to_robot(bot_id: str, skills: List[str], project_path: Path)
|
||||
skills_target_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Copying skills to {skills_target_dir}")
|
||||
|
||||
managed_skill_names = set()
|
||||
for base_dir in skills_source_dirs:
|
||||
if not base_dir.exists():
|
||||
continue
|
||||
for item in base_dir.iterdir():
|
||||
if item.is_dir():
|
||||
managed_skill_names.add(item.name)
|
||||
|
||||
# 清理不在列表中的多余 skill 文件夹
|
||||
expected_skill_names = {Path(skill.lstrip("@")).name for skill in skills}
|
||||
if skills_target_dir.exists():
|
||||
for item in skills_target_dir.iterdir():
|
||||
if item.is_dir() and item.name not in expected_skill_names:
|
||||
logger.info(f" Removing stale skill directory: {item}")
|
||||
if not item.is_dir() or item.name in expected_skill_names:
|
||||
continue
|
||||
if item.name in managed_skill_names:
|
||||
logger.info(f" Removing managed stale skill directory: {item}")
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
logger.info(f" Keeping unmanaged skill directory: {item}")
|
||||
|
||||
for skill in skills:
|
||||
skill_name = Path(skill.lstrip("@")).name
|
||||
target_dir = skills_target_dir / skill_name
|
||||
|
||||
# 如果目标目录已存在，跳过复制
|
||||
if target_dir.exists():
|
||||
logger.info(f" Skill '{skill}' already exists in {target_dir}, skipping")
|
||||
continue
|
||||
|
||||
source_dir = None
|
||||
|
||||
if skill.startswith("@"):
|
||||
@ -440,7 +447,7 @@ def _extract_skills_to_robot(bot_id: str, skills: List[str], project_path: Path)
|
||||
continue
|
||||
|
||||
try:
|
||||
shutil.copytree(source_dir, target_dir)
|
||||
logger.info(f" Copied: {source_dir} -> {target_dir}")
|
||||
shutil.copytree(source_dir, target_dir, dirs_exist_ok=True)
|
||||
logger.info(f" Synced: {source_dir} -> {target_dir}")
|
||||
except Exception as e:
|
||||
logger.error(f" Failed to copy {source_dir}: {e}")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user