Merge branch 'feature/rag_retrive_top_k' into developing
This commit is contained in:
朱潮
commit
a1a9d9fc47
@ -88,9 +88,8 @@ skill-name/
|
||||
|
||||
## Skill 加载优先级
|
||||
|
||||
1. Skill MCP 配置（最高）
|
||||
2. 默认 MCP 配置 (`mcp/mcp_settings.json`)
|
||||
3. 用户传入参数（覆盖所有）
|
||||
1. Skill MCP 配置
|
||||
2. 用户传入参数（覆盖已有同名配置）
|
||||
|
||||
## 安全措施
|
||||
|
||||
|
||||
@ -396,7 +396,6 @@ dataset_name/
|
||||
│ ├── document.txt # 原始文本内容
|
||||
│ ├── serialization.txt # 结构化数据
|
||||
│ └── schema.json # 字段定义和元数据
|
||||
├── mcp_settings.json # MCP 工具配置
|
||||
└── system_prompt.md # 系统提示词（可选）
|
||||
```
|
||||
|
||||
@ -405,7 +404,6 @@ dataset_name/
|
||||
- **document.txt**: 原始 Markdown 文本，提供完整上下文
|
||||
- **serialization.txt**: 格式化结构数据，每行 `字段1:值1;字段2:值2`
|
||||
- **schema.json**: 字段定义、枚举值映射和文件关联关系
|
||||
- **mcp_settings.json**: MCP 工具配置，定义可用的数据处理工具
|
||||
|
||||
---
|
||||
|
||||
@ -565,8 +563,7 @@ qwen-agent/
|
||||
│ ├── multi_keyword_search_server.py # 多关键词搜索服务
|
||||
│ ├── excel_csv_operator_server.py # Excel/CSV 操作服务
|
||||
│ ├── json_reader_server.py # JSON 读取服务
|
||||
│ ├── mcp_settings.json # MCP 配置文件
|
||||
│ └── tools/ # 工具定义文件
|
||||
│ └── tools/ # 工具定义文件
|
||||
├── models/ # 模型文件
|
||||
├── projects/ # 项目目录
|
||||
│ └── queue_data/ # 队列数据
|
||||
|
||||
@ -117,13 +117,6 @@ def read_system_prompt():
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def read_mcp_settings():
|
||||
"""读取MCP工具配置"""
|
||||
with open("./mcp/mcp_settings.json", "r") as f:
|
||||
mcp_settings_json = json.load(f)
|
||||
return mcp_settings_json
|
||||
|
||||
|
||||
async def get_tools_from_mcp(mcp):
|
||||
"""从MCP配置中提取工具（带缓存）"""
|
||||
start_time = time.time()
|
||||
@ -195,8 +188,7 @@ async def init_agent(config: AgentConfig):
|
||||
final_system_prompt = await load_system_prompt_async(config)
|
||||
final_mcp_settings = await load_mcp_settings_async(config)
|
||||
|
||||
# 如果没有提供mcp，使用config中的mcp_settings
|
||||
mcp_settings = final_mcp_settings if final_mcp_settings else read_mcp_settings()
|
||||
mcp_settings = final_mcp_settings if final_mcp_settings else []
|
||||
system_prompt = final_system_prompt if final_system_prompt else read_system_prompt()
|
||||
|
||||
config.system_prompt = mcp_settings
|
||||
|
||||
@ -38,8 +38,8 @@ class GuidelineMiddleware(AgentMiddleware):
|
||||
if not self.guidelines:
|
||||
self.guidelines = """
|
||||
1. General Inquiries
|
||||
Condition: User inquiries about products, policies, troubleshooting, factual questions, etc.
|
||||
Action: Priority given to invoking the 【Knowledge Base Retrieval】 tool to query the knowledge base.
|
||||
Condition: User inquiries about products, policies, troubleshooting, factual questions, definitions, workflows, data lookups, or other knowledge-seeking requests.
|
||||
Action: First choose the most suitable 【Knowledge Base Retrieval】 tool by scenario. Use table_rag_retrieve first for structured data, lists, statistics, comparisons, extraction, mixed requests, or unclear cases. Use rag_retrieve first only for clearly pure concept / definition / workflow / policy explanation questions. If the first retrieval result is empty, errored, irrelevant, or only partially answers the request, call the other retrieval tool before replying. Only reply that no relevant information was found after both retrieval tools have been tried and still provide no sufficient evidence.
|
||||
|
||||
2.Social Dialogue
|
||||
Condition: User intent involves small talk, greetings, expressions of thanks, compliments, or other non-substantive conversations.
|
||||
@ -47,7 +47,7 @@ Action: Provide concise, friendly, and personified natural responses.
|
||||
"""
|
||||
if not self.tool_description:
|
||||
self.tool_description = """
|
||||
- **Knowledge Base Retrieval**: For knowledge queries/other inquiries, prioritize searching the knowledge base → rag_retrieve-rag_retrieve
|
||||
- **Knowledge Base Retrieval**: Choose retrieval order by scenario. Default to `table_rag_retrieve -> rag_retrieve` for structured, list, mixed, or unclear requests. Use `rag_retrieve -> table_rag_retrieve` only for clearly pure concept or workflow questions. Do not answer with "no result" until both tools have been tried when retrieval is needed.
|
||||
"""
|
||||
|
||||
def get_guideline_prompt(self, config: AgentConfig) -> str:
|
||||
|
||||
@ -5,6 +5,7 @@ Claude Plugins 模式的 Hook 加载器
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import copy
|
||||
import logging
|
||||
import asyncio
|
||||
import subprocess
|
||||
@ -116,7 +117,8 @@ async def merge_skill_mcp_configs(bot_id: str) -> List[Dict]:
|
||||
plugin_config = json.load(f)
|
||||
servers = plugin_config.get('mcpServers', {})
|
||||
if servers:
|
||||
merged_servers.update(servers)
|
||||
normalized_servers = _normalize_skill_mcp_servers(servers, skill_path)
|
||||
merged_servers.update(normalized_servers)
|
||||
logger.info(f"Loaded MCP config from skill: {skill_name}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load mcpServers from {skill_name}: {e}")
|
||||
@ -127,6 +129,47 @@ async def merge_skill_mcp_configs(bot_id: str) -> List[Dict]:
|
||||
return []
|
||||
|
||||
|
||||
def _normalize_skill_mcp_servers(servers: Dict[str, Any], skill_path: str) -> Dict[str, Any]:
|
||||
"""将 skill plugin 中 stdio MCP server 的相对路径归一化为基于 skill 目录的绝对路径。"""
|
||||
normalized_servers = copy.deepcopy(servers)
|
||||
|
||||
for server_name, server_config in normalized_servers.items():
|
||||
if not isinstance(server_config, dict):
|
||||
continue
|
||||
|
||||
transport = server_config.get('transport')
|
||||
if not transport:
|
||||
transport = 'http' if 'url' in server_config else 'stdio'
|
||||
if transport != 'stdio':
|
||||
continue
|
||||
|
||||
command = server_config.get('command')
|
||||
if isinstance(command, str):
|
||||
server_config['command'] = _resolve_skill_relative_path(command, skill_path)
|
||||
|
||||
args = server_config.get('args')
|
||||
if isinstance(args, list):
|
||||
server_config['args'] = [
|
||||
_resolve_skill_relative_path(arg, skill_path) if isinstance(arg, str) else arg
|
||||
for arg in args
|
||||
]
|
||||
|
||||
return normalized_servers
|
||||
|
||||
|
||||
def _resolve_skill_relative_path(value: str, skill_path: str) -> str:
|
||||
"""将 ./ 或 ../ 开头且不含占位符的路径转为基于 skill 目录的绝对路径。"""
|
||||
if '{' in value or '}' in value:
|
||||
return value
|
||||
|
||||
if not value.startswith(('./', '../')):
|
||||
return value
|
||||
|
||||
normalized_path = os.path.abspath(os.path.join(skill_path, value))
|
||||
logger.debug(f"Resolved skill MCP path: {value} -> {normalized_path}")
|
||||
return normalized_path
|
||||
|
||||
|
||||
def _load_plugin_config(plugin_json_path: str) -> Dict:
|
||||
"""加载 plugin.json 配置"""
|
||||
try:
|
||||
|
||||
@ -203,10 +203,9 @@ async def load_mcp_settings_async(config) -> List[Dict]:
|
||||
List[Dict]: 合并后的MCP设置列表
|
||||
|
||||
Note:
|
||||
支持在 mcp_settings.json 的 args 中使用 {dataset_dir} 占位符，
|
||||
支持在传入或合并后的 mcp_settings 的 args 中使用 {dataset_dir} 占位符，
|
||||
会在 init_modified_agent_service_with_files 中被替换为实际的路径。
|
||||
"""
|
||||
from agent.config_cache import config_cache
|
||||
|
||||
# 从config中获取参数
|
||||
project_dir = getattr(config, 'project_dir', None)
|
||||
@ -222,33 +221,6 @@ async def load_mcp_settings_async(config) -> List[Dict]:
|
||||
skill_mcp_servers = skill_mcp_settings[0].get('mcpServers', {})
|
||||
logger.info(f"Loaded {len(skill_mcp_servers)} MCP servers from skills")
|
||||
# ===========================================================================================
|
||||
|
||||
# 2. 读取默认MCP设置（使用缓存）
|
||||
default_mcp_settings = []
|
||||
try:
|
||||
default_mcp_file = os.path.join("mcp", f"mcp_settings.json")
|
||||
default_mcp_settings = await config_cache.get_json_file(default_mcp_file) or []
|
||||
if default_mcp_settings:
|
||||
logger.info(f"Using cached default mcp_settings from mcp folder")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load default mcp_settings: {str(e)}")
|
||||
default_mcp_settings = []
|
||||
|
||||
# 3. 合并默认设置到merged_settings（默认设置被skill覆盖）
|
||||
if default_mcp_settings and len(default_mcp_settings) > 0:
|
||||
default_mcp_servers = default_mcp_settings[0].get('mcpServers', {})
|
||||
if merged_settings and len(merged_settings) > 0:
|
||||
# skill配置已存在，将默认配置合并进去（skill优先）
|
||||
skill_mcp_servers = merged_settings[0].get('mcpServers', {})
|
||||
# 默认配置中不存在的才添加
|
||||
for server_name, server_config in default_mcp_servers.items():
|
||||
if server_name not in skill_mcp_servers:
|
||||
skill_mcp_servers[server_name] = server_config
|
||||
else:
|
||||
# 没有skill配置，直接使用默认配置
|
||||
merged_settings = default_mcp_settings.copy()
|
||||
|
||||
# 遍历mcpServers工具，给每个工具增加env参数
|
||||
if merged_settings and len(merged_settings) > 0:
|
||||
mcp_servers = merged_settings[0].get('mcpServers', {})
|
||||
for server_name, server_config in mcp_servers.items():
|
||||
|
||||
@ -1,14 +0,0 @@
|
||||
[
|
||||
{
|
||||
"mcpServers": {
|
||||
"rag_retrieve": {
|
||||
"transport": "stdio",
|
||||
"command": "python",
|
||||
"args": [
|
||||
"./mcp/rag_retrieve_server.py",
|
||||
"{bot_id}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -1,35 +0,0 @@
|
||||
[
|
||||
{
|
||||
"name": "rag_retrieve",
|
||||
"description": "Retrieve relevant documents from the knowledge base. Returns markdown format results containing relevant content.\n\n[CALLING STRATEGY] This tool is the SECONDARY choice. Only call this tool FIRST when the question is clearly a pure knowledge/concept query (e.g. \"What does XX mean?\", \"How to use XX?\", \"What is the workflow for XX?\") that has NO relation to data, lists, summaries, or tabular output. In ALL other cases, call table_rag_retrieve FIRST, then use this tool to supplement if table_rag results are insufficient or need additional context.\n\n[PARAMETER USAGE — IMPORTANT]\n- Do NOT pass the user's raw question directly unless it already fits retrieval needs well.\n- You MUST construct `query` by following the rewriting strategy described in the `query` parameter schema below.\n- You MUST choose `top_k` dynamically by following the decision rules described in the `top_k` parameter schema below.\n- `query` and `top_k` are coupled: broader rewritten queries, list-style requests, historical coverage, or multi-branch recall usually require a higher `top_k`; narrow fact lookup should use a smaller `top_k`.\n- Prefer the smallest sufficient `top_k`, then expand only when coverage is insufficient.\n\n[WHEN TO USE AS SUPPLEMENT] After calling table_rag_retrieve, call this tool if:\n- table_rag_retrieve returned insufficient results and you need document context\n- The answer requires background explanation beyond the structured data\n- The user's question involves both data retrieval and conceptual understanding",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Retrieval query content. Before retrieval, rewrite the query to improve recall: extract the core entity, time scope, attributes, and intent; add meaningful variants such as synonyms, aliases, abbreviations, related titles, historical names, and category terms; expand enumeration-style queries more aggressively; preserve the original meaning and do not introduce unrelated topics; use both the original query and rewritten queries whenever possible. For historical or list-style queries, also add terms like title / organization variants, predecessor / successor, former / past / historical / all-time, and list / overview / roster / timeline / archive."
|
||||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"description": "Number of top results to retrieve. Use the smallest sufficient top_k and expand only when coverage is insufficient: 30 for simple fact lookup about one specific thing; 50 for moderate synthesis, comparison, summarization, or disambiguation; 100 for broad-recall queries needing high coverage, such as comprehensive analysis, scattered knowledge, multiple entities or periods, list / catalog / timeline / roster / overview requests, or all items / historical succession / many records. Raise top_k when query rewrite produces many useful keyword branches or when results are too few, repetitive, incomplete, sparse, or too narrow in coverage. Do not raise top_k just because the query is longer. Expansion sequence: 30 -> 50 -> 100. If uncertain, prefer passing 100. Default: 100.",
|
||||
"default": 100
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "table_rag_retrieve",
|
||||
"description": "Retrieve relevant data from Excel/spreadsheet files in the knowledge base. Returns markdown format results containing table data analysis.\n\n[CALLING STRATEGY] This tool is the DEFAULT first choice. Call this tool FIRST in any of the following situations:\n- Questions involving specific values, prices, quantities, inventory, specifications, rankings, comparisons, statistics\n- Requests for tabular output (e.g. \"make a table\", \"list in a table\", \"一覧表にして\", \"整理成表格\")\n- Information extraction/organization requests (e.g. \"extract\", \"list\", \"summarize and list\", \"抽出\", \"提取\", \"列举\", \"汇总\")\n- Queries about specific person names, project names, or product names (e.g. \"XX議員の答弁を一覧にして\")\n- ANY question where you are unsure whether table data is needed — default to calling this tool first\n\n[PARAMETER USAGE — IMPORTANT]\n- Do NOT pass the user's raw question directly unless it already fits retrieval needs well.\n- You MUST construct `query` by following the rewriting strategy described in the `query` parameter schema below.\n- For list, extraction, comparison, historical, or name-based requests, rewrite `query` more aggressively to cover entity variants and intent variants.\n\n[RESPONSE HANDLING] When processing the returned results:\n1. Follow all instructions in [INSTRUCTION] and [EXTRA_INSTRUCTION] sections of the response (e.g. output format, source citation requirements)\n2. If Query result hint indicates truncation (e.g. \"Only the first N rows are included; the remaining M rows were omitted\"), you MUST explicitly tell the user: total matches (N+M), displayed count (N), and omitted count (M)\n3. If query result is empty, respond truthfully that no relevant data was found — do NOT fabricate data\n4. Cite data sources using file names from file_ref_table in the response",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Retrieval query content for table data. Before retrieval, rewrite the query to improve recall: extract the core entity, time scope, attributes, and intent; add meaningful variants such as synonyms, aliases, abbreviations, related titles, historical names, and category terms; expand enumeration-style queries more aggressively; preserve the original meaning and do not introduce unrelated topics; use both the original query and rewritten queries whenever possible. For historical or list-style queries, also add terms like title / organization variants, predecessor / successor, former / past / historical / all-time, and list / overview / roster / timeline / archive."
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
]
|
||||
@ -1,17 +1,6 @@
|
||||
{extra_prompt}
|
||||
|
||||
## CITATION REQUIREMENTS
|
||||
|
||||
When your answer uses learned knowledge, you MUST generate `<CITATION ... />` tags. Follow the specific citation format instructions returned by each tool (`rag_retrieve`, `table_rag_retrieve`).
|
||||
|
||||
### General Placement Rules
|
||||
1. Citations MUST appear IMMEDIATELY AFTER the paragraph or bullet list that uses the knowledge
|
||||
2. NEVER collect all citations and place them at the end of your response
|
||||
3. Limit to 1-2 citations per paragraph/bullet list - combine related facts under one citation
|
||||
4. If your answer uses learned knowledge, you MUST generate at least 1 `<CITATION ... />` in the response
|
||||
|
||||
### Current Working Directory
|
||||
|
||||
# Current Working Directory
|
||||
PROJECT_ROOT: `{agent_dir_path}`
|
||||
The filesystem backend is currently operating in: `{agent_dir_path}`
|
||||
|
||||
@ -80,7 +69,7 @@ When creating scripts in `executable_code/`, follow these organization rules:
|
||||
- Temporary script (when needed): `{agent_dir_path}/executable_code/tmp/test.py`
|
||||
- Downloaded file: `{agent_dir_path}/download/report.pdf`
|
||||
|
||||
## System Information
|
||||
# System Information
|
||||
<env>
|
||||
Working directory: {agent_dir_path}
|
||||
Current User: {user_identifier}
|
||||
@ -90,10 +79,7 @@ Trace Id: {trace_id}
|
||||
|
||||
# Execution Guidelines
|
||||
- **Tool-Driven**: All operations are implemented through tool interfaces.
|
||||
- **Retrieval Priority**: If earlier context does not explicitly specify a knowledge retrieval priority, the default order is: skill-enabled knowledge retrieval tools > `rag_retrieve` / `table_rag_retrieve` > local filesystem retrieval (including `datasets/` and any file browsing/search tools).
|
||||
- **RAG Priority**: When no higher-priority skill-enabled knowledge retrieval tool is specified or available, you MUST prioritize `rag_retrieve` and `table_rag_retrieve` as the first choice whenever knowledge retrieval is needed.
|
||||
- **Filesystem Last**: The local filesystem is the lowest-priority source. Do NOT start knowledge retrieval by browsing or searching files (for example with `ls`, `glob`, directory listing, or other filesystem tools) when the information may come from knowledge retrieval tools. Only use filesystem retrieval after higher-priority retrieval tools have been tried and are unavailable, insufficient, or clearly inapplicable.
|
||||
- **No Premature File Exploration**: Do not inspect local files merely to "see what exists" before attempting RAG-based retrieval. File inspection is a fallback, not the default path.
|
||||
- **No Premature File Exploration**: Do not inspect local files merely to "see what exists" before attempting earlier knowledge retrieval sources. Local filesystem retrieval is the final fallback, not the default path, but do not skip it when earlier retrieval sources are insufficient.
|
||||
- **Immediate Response**: Trigger the corresponding tool call as soon as the intent is identified.
|
||||
- **Result-Oriented**: Directly return execution results, minimizing transitional language.
|
||||
- **Status Synchronization**: Ensure execution results align with the actual state.
|
||||
|
||||
22
skills/rag-retrieve-only/.claude-plugin/plugin.json
Normal file
22
skills/rag-retrieve-only/.claude-plugin/plugin.json
Normal file
@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "rag-retrieve-only",
|
||||
"description": "Only provides rag_retrieve. table_rag_retrieve and local file retrieval are disabled.",
|
||||
"hooks": {
|
||||
"PrePrompt": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "python hooks/pre_prompt.py"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mcpServers": {
|
||||
"rag_retrieve": {
|
||||
"transport": "stdio",
|
||||
"command": "python",
|
||||
"args": [
|
||||
"./rag_retrieve_server.py",
|
||||
"{bot_id}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
34
skills/rag-retrieve-only/README.md
Normal file
34
skills/rag-retrieve-only/README.md
Normal file
@ -0,0 +1,34 @@
|
||||
# rag-retrieve
|
||||
|
||||
只保留 `rag_retrieve` 的精简版插件示例。
|
||||
|
||||
## 功能说明
|
||||
|
||||
- 通过 `PrePrompt` Hook 注入检索策略
|
||||
- 暴露 `rag_retrieve` MCP Server
|
||||
- 插件仅支持 `rag_retrieve`
|
||||
- 已禁用 `table_rag_retrieve`
|
||||
- 已禁用本地文件检索
|
||||
|
||||
## 目录结构
|
||||
|
||||
```text
|
||||
rag-retrieve-only/
|
||||
├── README.md
|
||||
├── .claude-plugin/
|
||||
│ └── plugin.json
|
||||
├── hooks/
|
||||
│ ├── pre_prompt.py
|
||||
│ └── retrieval-policy.md
|
||||
├── rag_retrieve_server.py
|
||||
└── rag_retrieve_tools.json
|
||||
```
|
||||
|
||||
## 当前检索策略
|
||||
|
||||
默认顺序：skill-enabled knowledge retrieval tools > `rag_retrieve`
|
||||
|
||||
- 优先使用可用的技能内知识检索工具
|
||||
- 不足时使用 `rag_retrieve`
|
||||
- 不并行执行多个检索源
|
||||
- 插件仅支持 `rag_retrieve`
|
||||
20
skills/rag-retrieve-only/hooks/pre_prompt.py
Normal file
20
skills/rag-retrieve-only/hooks/pre_prompt.py
Normal file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PreMemoryPrompt Hook - 用户上下文加载器示例
|
||||
|
||||
在记忆提取提示词（FACT_RETRIEVAL_PROMPT）加载时执行，
|
||||
读取同目录下的 memory_prompt.md 作为自定义记忆提取提示词模板。
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
prompt_file = Path(__file__).parent / "retrieval-policy.md"
|
||||
if prompt_file.exists():
|
||||
print(prompt_file.read_text(encoding="utf-8"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
31
skills/rag-retrieve-only/hooks/retrieval-policy.md
Normal file
31
skills/rag-retrieve-only/hooks/retrieval-policy.md
Normal file
@ -0,0 +1,31 @@
|
||||
# Retrieval Policy
|
||||
|
||||
- `rag_retrieve` is the only knowledge source.
|
||||
- Do NOT answer from model knowledge first.
|
||||
|
||||
## 1.Query Preparation
|
||||
- Do NOT pass the raw user question unless it already works well for retrieval.
|
||||
- Rewrite for recall: extract entity, time scope, attributes, and intent.
|
||||
- Add useful variants: synonyms, aliases, abbreviations, related titles, historical names, and category terms.
|
||||
- Expand list-style, extraction, overview, historical, roster, timeline, and archive queries more aggressively.
|
||||
- Preserve meaning. Do NOT introduce unrelated topics.
|
||||
|
||||
## 2.Retrieval Breadth (`top_k`)
|
||||
- Apply `top_k` only to `rag_retrieve`. Use the smallest sufficient value, then expand only if coverage is insufficient.
|
||||
- Use `30` for simple fact lookup.
|
||||
- Use `50` for moderate synthesis, comparison, summarization, or disambiguation.
|
||||
- Use `100` for broad recall, such as comprehensive analysis, scattered knowledge, multiple entities or periods, or list / catalog / timeline / roster / overview requests.
|
||||
- Raise `top_k` when keyword branches are many or results are too few, repetitive, incomplete, sparse, or too narrow.
|
||||
- Use this expansion order: `30 -> 50 -> 100`. If unsure, use `100`.
|
||||
|
||||
## 3.Retry
|
||||
- If the result is insufficient, retry `rag_retrieve` with a better rewritten query or a larger `top_k`.
|
||||
- Only say no relevant information was found after `rag_retrieve` has been tried and still provides insufficient evidence.
|
||||
|
||||
## 4.Citation Requirements for Retrieved Knowledge
|
||||
- When using knowledge from `rag_retrieve`, you MUST generate `<CITATION ... />` tags.
|
||||
- Follow the citation format returned by each tool.
|
||||
- Place citations immediately after the paragraph or bullet list that uses the knowledge.
|
||||
- Do NOT collect citations at the end.
|
||||
- Use 1-2 citations per paragraph or bullet list when possible.
|
||||
- If learned knowledge is used, include at least 1 `<CITATION ... />`.
|
||||
251
skills/rag-retrieve-only/mcp_common.py
Normal file
251
skills/rag-retrieve-only/mcp_common.py
Normal file
@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MCP服务器通用工具函数
|
||||
提供路径处理、文件验证、请求处理等公共功能
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
import re
|
||||
|
||||
def get_allowed_directory():
|
||||
"""获取允许访问的目录"""
|
||||
# 优先使用命令行参数传入的dataset_dir
|
||||
if len(sys.argv) > 1:
|
||||
dataset_dir = sys.argv[1]
|
||||
return os.path.abspath(dataset_dir)
|
||||
|
||||
# 从环境变量读取项目数据目录
|
||||
project_dir = os.getenv("PROJECT_DATA_DIR", "./projects/data")
|
||||
return os.path.abspath(project_dir)
|
||||
|
||||
|
||||
def resolve_file_path(file_path: str, default_subfolder: str = "default") -> str:
|
||||
"""
|
||||
解析文件路径，支持 folder/document.txt 和 document.txt 两种格式
|
||||
|
||||
Args:
|
||||
file_path: 输入的文件路径
|
||||
default_subfolder: 当只传入文件名时使用的默认子文件夹名称
|
||||
|
||||
Returns:
|
||||
解析后的完整文件路径
|
||||
"""
|
||||
# 如果路径包含文件夹分隔符，直接使用
|
||||
if '/' in file_path or '\\' in file_path:
|
||||
clean_path = file_path.replace('\\', '/')
|
||||
|
||||
# 移除 projects/ 前缀（如果存在）
|
||||
if clean_path.startswith('projects/'):
|
||||
clean_path = clean_path[9:] # 移除 'projects/' 前缀
|
||||
elif clean_path.startswith('./projects/'):
|
||||
clean_path = clean_path[11:] # 移除 './projects/' 前缀
|
||||
else:
|
||||
# 如果只有文件名，添加默认子文件夹
|
||||
clean_path = f"{default_subfolder}/{file_path}"
|
||||
|
||||
# 获取允许的目录
|
||||
project_data_dir = get_allowed_directory()
|
||||
|
||||
# 尝试在项目目录中查找文件
|
||||
full_path = os.path.join(project_data_dir, clean_path.lstrip('./'))
|
||||
if os.path.exists(full_path):
|
||||
return full_path
|
||||
|
||||
# 如果直接路径不存在，尝试递归查找
|
||||
found = find_file_in_project(clean_path, project_data_dir)
|
||||
if found:
|
||||
return found
|
||||
|
||||
# 如果是纯文件名且在default子文件夹中不存在，尝试在根目录查找
|
||||
if '/' not in file_path and '\\' not in file_path:
|
||||
root_path = os.path.join(project_data_dir, file_path)
|
||||
if os.path.exists(root_path):
|
||||
return root_path
|
||||
|
||||
raise FileNotFoundError(f"File not found: {file_path} (searched in {project_data_dir})")
|
||||
|
||||
|
||||
def find_file_in_project(filename: str, project_dir: str) -> Optional[str]:
|
||||
"""在项目目录中递归查找文件"""
|
||||
# 如果filename包含路径，只搜索指定的路径
|
||||
if '/' in filename:
|
||||
parts = filename.split('/')
|
||||
target_file = parts[-1]
|
||||
search_dir = os.path.join(project_dir, *parts[:-1])
|
||||
|
||||
if os.path.exists(search_dir):
|
||||
target_path = os.path.join(search_dir, target_file)
|
||||
if os.path.exists(target_path):
|
||||
return target_path
|
||||
else:
|
||||
# 纯文件名，递归搜索整个项目目录
|
||||
for root, dirs, files in os.walk(project_dir):
|
||||
if filename in files:
|
||||
return os.path.join(root, filename)
|
||||
return None
|
||||
|
||||
|
||||
def load_tools_from_json(tools_file_name: str) -> List[Dict[str, Any]]:
|
||||
"""从 JSON 文件加载工具定义"""
|
||||
try:
|
||||
tools_file = os.path.join(os.path.dirname(__file__), tools_file_name)
|
||||
if os.path.exists(tools_file):
|
||||
with open(tools_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
# 如果 JSON 文件不存在，使用默认定义
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"Warning: Unable to load tool definition JSON file: {str(e)}")
|
||||
return []
|
||||
|
||||
|
||||
def create_error_response(request_id: Any, code: int, message: str) -> Dict[str, Any]:
|
||||
"""创建标准化的错误响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_success_response(request_id: Any, result: Any) -> Dict[str, Any]:
|
||||
"""创建标准化的成功响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": result
|
||||
}
|
||||
|
||||
|
||||
def create_initialize_response(request_id: Any, server_name: str, server_version: str = "1.0.0") -> Dict[str, Any]:
|
||||
"""创建标准化的初始化响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": server_name,
|
||||
"version": server_version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_ping_response(request_id: Any) -> Dict[str, Any]:
|
||||
"""创建标准化的ping响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"pong": True
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_tools_list_response(request_id: Any, tools: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""创建标准化的工具列表响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"tools": tools
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def is_regex_pattern(pattern: str) -> bool:
|
||||
"""检测字符串是否为正则表达式模式"""
|
||||
# 检查 /pattern/ 格式
|
||||
if pattern.startswith('/') and pattern.endswith('/') and len(pattern) > 2:
|
||||
return True
|
||||
|
||||
# 检查 r"pattern" 或 r'pattern' 格式
|
||||
if pattern.startswith(('r"', "r'")) and pattern.endswith(('"', "'")) and len(pattern) > 3:
|
||||
return True
|
||||
|
||||
# 检查是否包含正则特殊字符
|
||||
regex_chars = {'*', '+', '?', '|', '(', ')', '[', ']', '{', '}', '^', '$', '\\', '.'}
|
||||
return any(char in pattern for char in regex_chars)
|
||||
|
||||
|
||||
def compile_pattern(pattern: str) -> Union[re.Pattern, str, None]:
|
||||
"""编译正则表达式模式，如果不是正则则返回原字符串"""
|
||||
if not is_regex_pattern(pattern):
|
||||
return pattern
|
||||
|
||||
try:
|
||||
# 处理 /pattern/ 格式
|
||||
if pattern.startswith('/') and pattern.endswith('/'):
|
||||
regex_body = pattern[1:-1]
|
||||
return re.compile(regex_body)
|
||||
|
||||
# 处理 r"pattern" 或 r'pattern' 格式
|
||||
if pattern.startswith(('r"', "r'")) and pattern.endswith(('"', "'")):
|
||||
regex_body = pattern[2:-1]
|
||||
return re.compile(regex_body)
|
||||
|
||||
# 直接编译包含正则字符的字符串
|
||||
return re.compile(pattern)
|
||||
except re.error as e:
|
||||
# 如果编译失败，返回None表示无效的正则
|
||||
print(f"Warning: Regular expression '{pattern}' compilation failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
async def handle_mcp_streaming(request_handler):
|
||||
"""处理MCP请求的标准主循环"""
|
||||
try:
|
||||
while True:
|
||||
# Read from stdin
|
||||
line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline)
|
||||
if not line:
|
||||
break
|
||||
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
request = json.loads(line)
|
||||
response = await request_handler(request)
|
||||
|
||||
# Write to stdout
|
||||
sys.stdout.write(json.dumps(response, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
except json.JSONDecodeError:
|
||||
error_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"error": {
|
||||
"code": -32700,
|
||||
"message": "Parse error"
|
||||
}
|
||||
}
|
||||
sys.stdout.write(json.dumps(error_response, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
except Exception as e:
|
||||
error_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"error": {
|
||||
"code": -32603,
|
||||
"message": f"Internal error: {str(e)}"
|
||||
}
|
||||
}
|
||||
sys.stdout.write(json.dumps(error_response, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
222
skills/rag-retrieve-only/rag_retrieve_server.py
Normal file
222
skills/rag-retrieve-only/rag_retrieve_server.py
Normal file
@ -0,0 +1,222 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
RAG检索MCP服务器
|
||||
调用本地RAG API进行文档检索
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
from typing import Any, Dict
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("Error: requests module is required. Please install it with: pip install requests")
|
||||
sys.exit(1)
|
||||
|
||||
from mcp_common import (
|
||||
create_error_response,
|
||||
create_success_response,
|
||||
create_initialize_response,
|
||||
create_ping_response,
|
||||
create_tools_list_response,
|
||||
load_tools_from_json,
|
||||
handle_mcp_streaming
|
||||
)
|
||||
BACKEND_HOST = os.getenv("BACKEND_HOST", "https://api-dev.gptbase.ai")
|
||||
MASTERKEY = os.getenv("MASTERKEY", "master")
|
||||
|
||||
# Citation instruction prefixes injected into tool results
|
||||
DOCUMENT_CITATION_INSTRUCTIONS = """<CITATION_INSTRUCTIONS>
|
||||
When using the retrieved knowledge below, you MUST add XML citation tags for factual claims.
|
||||
|
||||
## Document Knowledge
|
||||
Format: `<CITATION file="file_uuid" filename="name.pdf" page=3 />`
|
||||
- Use `file` attribute with the UUID from document markers
|
||||
- Use `filename` attribute with the actual filename from document markers
|
||||
- Use `page` attribute (singular) with the page number
|
||||
- `page` MUST be 0-based and must match the `pages:` values shown in the learned knowledge context
|
||||
|
||||
## Web Page Knowledge
|
||||
Format: `<CITATION url="https://example.com/page" />`
|
||||
- Use `url` attribute with the web page URL from the source metadata
|
||||
- Do not use `file`, `filename`, or `page` attributes for web sources
|
||||
- If content is grounded in a web source, prefer a web citation with `url` over a file citation
|
||||
|
||||
## Placement Rules
|
||||
- Citations MUST appear IMMEDIATELY AFTER the paragraph or bullet list that uses the knowledge
|
||||
- NEVER collect all citations and place them at the end of your response
|
||||
- Limit to 1-2 citations per paragraph/bullet list
|
||||
- If your answer uses learned knowledge, you MUST generate at least 1 `<CITATION ... />` in the response
|
||||
</CITATION_INSTRUCTIONS>
|
||||
|
||||
"""
|
||||
|
||||
def rag_retrieve(query: str, top_k: int = 100) -> Dict[str, Any]:
|
||||
"""调用RAG检索API"""
|
||||
try:
|
||||
bot_id = ""
|
||||
if len(sys.argv) > 1:
|
||||
bot_id = sys.argv[1]
|
||||
|
||||
url = f"{BACKEND_HOST}/v1/rag_retrieve/{bot_id}"
|
||||
if not url:
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Error: RAG API URL not provided. Please provide URL as command line argument."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
masterkey = MASTERKEY
|
||||
token_input = f"{masterkey}:{bot_id}"
|
||||
auth_token = hashlib.md5(token_input.encode()).hexdigest()
|
||||
|
||||
headers = {
|
||||
"content-type": "application/json",
|
||||
"authorization": f"Bearer {auth_token}"
|
||||
}
|
||||
data = {
|
||||
"query": query,
|
||||
"top_k": top_k
|
||||
}
|
||||
|
||||
response = requests.post(url, json=data, headers=headers, timeout=30)
|
||||
|
||||
if response.status_code != 200:
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Error: RAG API returned status code {response.status_code}. Response: {response.text}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
try:
|
||||
response_data = response.json()
|
||||
except json.JSONDecodeError as e:
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Error: Failed to parse API response as JSON. Error: {str(e)}, Raw response: {response.text}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
if "markdown" in response_data:
|
||||
markdown_content = response_data["markdown"]
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": DOCUMENT_CITATION_INSTRUCTIONS + markdown_content
|
||||
}
|
||||
]
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Error: 'markdown' field not found in API response. Response: {json.dumps(response_data, indent=2, ensure_ascii=False)}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Error: Failed to connect to RAG API. {str(e)}"
|
||||
}
|
||||
]
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Error: {str(e)}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
async def handle_request(request: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Handle MCP request"""
|
||||
try:
|
||||
method = request.get("method")
|
||||
params = request.get("params", {})
|
||||
request_id = request.get("id")
|
||||
|
||||
if method == "initialize":
|
||||
return create_initialize_response(request_id, "rag-retrieve")
|
||||
|
||||
elif method == "ping":
|
||||
return create_ping_response(request_id)
|
||||
|
||||
elif method == "tools/list":
|
||||
tools = load_tools_from_json("rag_retrieve_tools.json")
|
||||
if not tools:
|
||||
tools = [
|
||||
{
|
||||
"name": "rag_retrieve",
|
||||
"description": "调用RAG检索API，根据查询内容检索相关文档。返回包含相关内容的markdown格式结果。",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "检索查询内容"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
]
|
||||
return create_tools_list_response(request_id, tools)
|
||||
|
||||
elif method == "tools/call":
|
||||
tool_name = params.get("name")
|
||||
arguments = params.get("arguments", {})
|
||||
|
||||
if tool_name == "rag_retrieve":
|
||||
query = arguments.get("query", "")
|
||||
top_k = arguments.get("top_k", 100)
|
||||
|
||||
if not query:
|
||||
return create_error_response(request_id, -32602, "Missing required parameter: query")
|
||||
|
||||
result = rag_retrieve(query, top_k)
|
||||
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": result
|
||||
}
|
||||
|
||||
else:
|
||||
return create_error_response(request_id, -32601, f"Unknown tool: {tool_name}")
|
||||
|
||||
else:
|
||||
return create_error_response(request_id, -32601, f"Unknown method: {method}")
|
||||
|
||||
except Exception as e:
|
||||
return create_error_response(request.get("id"), -32603, f"Internal error: {str(e)}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Main entry point."""
|
||||
await handle_mcp_streaming(handle_request)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
21
skills/rag-retrieve-only/rag_retrieve_tools.json
Normal file
21
skills/rag-retrieve-only/rag_retrieve_tools.json
Normal file
@ -0,0 +1,21 @@
|
||||
[
|
||||
{
|
||||
"name": "rag_retrieve",
|
||||
"description": "Retrieve relevant documents from the knowledge base. Returns markdown results. Use this tool for concept, definition, workflow, policy, explanation, and general knowledge lookup. Rewrite the query when needed to improve recall.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Retrieval query content. Rewrite the query when needed to improve recall."
|
||||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"description": "Number of top results to retrieve. Choose dynamically based on retrieval breadth and coverage needs.",
|
||||
"default": 100
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
]
|
||||
22
skills_autoload/rag-retrieve/.claude-plugin/plugin.json
Normal file
22
skills_autoload/rag-retrieve/.claude-plugin/plugin.json
Normal file
@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "rag-retrieve",
|
||||
"description": "Provides RAG and table RAG retrieval tools through a PrePrompt hook and MCP server.",
|
||||
"hooks": {
|
||||
"PrePrompt": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "python hooks/pre_prompt.py"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mcpServers": {
|
||||
"rag_retrieve": {
|
||||
"transport": "stdio",
|
||||
"command": "python",
|
||||
"args": [
|
||||
"./rag_retrieve_server.py",
|
||||
"{bot_id}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
99
skills_autoload/rag-retrieve/README.md
Normal file
99
skills_autoload/rag-retrieve/README.md
Normal file
@ -0,0 +1,99 @@
|
||||
# RAG Retrieve
|
||||
|
||||
An example autoload skill that demonstrates how to integrate `rag-retrieve` and `table-rag-retrieve` through Claude Plugins hooks and an MCP server.
|
||||
|
||||
## Overview
|
||||
|
||||
This skill uses a `PrePrompt` hook to inject retrieval guidance into the prompt, and starts an MCP server that exposes retrieval capabilities for the current bot.
|
||||
|
||||
### PrePrompt Hook
|
||||
Runs when the system prompt is loaded and injects retrieval policy content.
|
||||
- File: `hooks/pre_prompt.py`
|
||||
- Purpose: load retrieval instructions and add them to the prompt context
|
||||
|
||||
### MCP Server
|
||||
Provides retrieval tools over stdio for the current `bot_id`.
|
||||
- File: `rag_retrieve_server.py`
|
||||
- Purpose: expose `rag-retrieve` and related retrieval tools to the agent
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```text
|
||||
rag-retrieve/
|
||||
├── README.md # Skill documentation
|
||||
├── .claude-plugin/
|
||||
│ └── plugin.json # Hook and MCP server configuration
|
||||
├── hooks/
|
||||
│ ├── pre_prompt.py # PrePrompt hook script
|
||||
│ └── retrieval-policy.md # Retrieval policy injected into the prompt
|
||||
├── mcp_common.py # Shared MCP utilities
|
||||
├── rag_retrieve_server.py # MCP server entrypoint
|
||||
└── rag_retrieve_tools.json # Tool definitions
|
||||
```
|
||||
|
||||
## `plugin.json` Format
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "rag-retrieve",
|
||||
"description": "rag-retrieve and table-rag-retrieve",
|
||||
"hooks": {
|
||||
"PrePrompt": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "python hooks/pre_prompt.py"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mcpServers": {
|
||||
"rag_retrieve": {
|
||||
"transport": "stdio",
|
||||
"command": "python",
|
||||
"args": [
|
||||
"./skills_autoload/rag-retrieve/rag_retrieve_server.py",
|
||||
"{bot_id}"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Hook Script Behavior
|
||||
|
||||
The hook script runs as a subprocess, receives input through environment variables, and writes the injected content to stdout.
|
||||
|
||||
### Available Environment Variables
|
||||
|
||||
| Environment Variable | Description | Applies To |
|
||||
|----------------------|-------------|------------|
|
||||
| `ASSISTANT_ID` | Bot ID | All hooks |
|
||||
| `USER_IDENTIFIER` | User identifier | All hooks |
|
||||
| `SESSION_ID` | Session ID | All hooks |
|
||||
| `LANGUAGE` | Language code | All hooks |
|
||||
| `HOOK_TYPE` | Hook type | All hooks |
|
||||
|
||||
### PrePrompt Example
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
def main():
|
||||
user_identifier = os.environ.get('USER_IDENTIFIER', '')
|
||||
bot_id = os.environ.get('ASSISTANT_ID', '')
|
||||
|
||||
print(f"## Retrieval Context\n\nUser: {user_identifier}\nBot: {bot_id}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
```
|
||||
|
||||
## Example Use Cases
|
||||
|
||||
1. **Prompt-time retrieval guidance**: inject retrieval rules before the model starts reasoning
|
||||
2. **Bot-specific retrieval setup**: start the MCP server with the current `bot_id`
|
||||
3. **Unified retrieval access**: expose RAG and table RAG tools through a single skill
|
||||
20
skills_autoload/rag-retrieve/hooks/pre_prompt.py
Normal file
20
skills_autoload/rag-retrieve/hooks/pre_prompt.py
Normal file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PreMemoryPrompt Hook - 用户上下文加载器示例
|
||||
|
||||
在记忆提取提示词（FACT_RETRIEVAL_PROMPT）加载时执行，
|
||||
读取同目录下的 memory_prompt.md 作为自定义记忆提取提示词模板。
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
prompt_file = Path(__file__).parent / "retrieval-policy.md"
|
||||
if prompt_file.exists():
|
||||
print(prompt_file.read_text(encoding="utf-8"))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
55
skills_autoload/rag-retrieve/hooks/retrieval-policy.md
Normal file
55
skills_autoload/rag-retrieve/hooks/retrieval-policy.md
Normal file
@ -0,0 +1,55 @@
|
||||
# Retrieval Policy
|
||||
|
||||
### 1. Retrieval Order and Tool Selection
|
||||
- Follow this section for source choice, tool choice, query rewrite, `top_k`, fallback, result handling, and citations.
|
||||
- Use this default retrieval order and execute it sequentially: skill-enabled knowledge retrieval tools > `rag_retrieve` / `table_rag_retrieve`.
|
||||
- Do NOT answer from model knowledge first.
|
||||
- Do NOT bypass the retrieval flow and inspect local filesystem documents on your own.
|
||||
- Do NOT use local filesystem retrieval as a fallback knowledge source.
|
||||
- Local filesystem documents are not a recommended retrieval source here because file formats are inconsistent and have not been normalized or parsed for reliable knowledge lookup.
|
||||
- Knowledge must be retrieved through the supported knowledge tools only: skill-enabled retrieval scripts, `table_rag_retrieve`, and `rag_retrieve`.
|
||||
- When a suitable skill-enabled knowledge retrieval tool is available, use it first.
|
||||
- If no suitable skill-enabled retrieval tool is available, or if its result is insufficient, continue with `rag_retrieve` or `table_rag_retrieve`.
|
||||
- Use `table_rag_retrieve` first for values, prices, quantities, inventory, specifications, rankings, comparisons, summaries, extraction, lists, tables, name lookup, historical coverage, mixed questions, and unclear cases.
|
||||
- Use `rag_retrieve` first only for clearly pure concept, definition, workflow, policy, or explanation questions without structured data needs.
|
||||
- After each retrieval step, evaluate sufficiency before moving to the next source. Do NOT run these retrieval sources in parallel.
|
||||
|
||||
### 2. Query Preparation
|
||||
- Do NOT pass the raw user question unless it already works well for retrieval.
|
||||
- Rewrite for recall: extract entity, time scope, attributes, and intent.
|
||||
- Add useful variants: synonyms, aliases, abbreviations, related titles, historical names, and category terms.
|
||||
- Expand list-style, extraction, overview, historical, roster, timeline, and archive queries more aggressively.
|
||||
- Preserve meaning. Do NOT introduce unrelated topics.
|
||||
|
||||
### 3. Retrieval Breadth (`top_k`)
|
||||
- Apply `top_k` only to `rag_retrieve`. Use the smallest sufficient value, then expand only if coverage is insufficient.
|
||||
- Use `30` for simple fact lookup.
|
||||
- Use `50` for moderate synthesis, comparison, summarization, or disambiguation.
|
||||
- Use `100` for broad recall, such as comprehensive analysis, scattered knowledge, multiple entities or periods, or list / catalog / timeline / roster / overview requests.
|
||||
- Raise `top_k` when keyword branches are many or results are too few, repetitive, incomplete, sparse, or too narrow.
|
||||
- Use this expansion order: `30 -> 50 -> 100`. If unsure, use `100`.
|
||||
|
||||
### 4. Result Evaluation
|
||||
- Treat results as insufficient if they are empty, start with `Error:`, say `no excel files found`, are off-topic, miss the core entity or scope, or provide no usable evidence.
|
||||
- Also treat results as insufficient when they cover only part of the request, or when full-list, historical, comparison, or mixed data + explanation requests return only partial or truncated coverage.
|
||||
|
||||
### 5. Fallback and Sequential Retry
|
||||
- If the first retrieval result is insufficient, call the next supported retrieval source in the default order before replying.
|
||||
- `table_rag_retrieve` now performs an internal fallback to `rag_retrieve` when it returns `no excel files found`, but this does NOT change the higher-level retrieval order.
|
||||
- If `table_rag_retrieve` is still insufficient after its internal fallback, continue with the next supported retrieval source instead of checking local filesystem documents directly.
|
||||
- If `rag_retrieve` is insufficient or empty, continue with `table_rag_retrieve`.
|
||||
- Say no relevant information was found only after all applicable skill-enabled retrieval tools, `rag_retrieve`, and `table_rag_retrieve` have been tried and still do not provide enough evidence.
|
||||
- Do NOT reply that no relevant information was found before the supported knowledge retrieval flow has been exhausted.
|
||||
|
||||
### 6. Table RAG Result Handling
|
||||
- Follow all `[INSTRUCTION]` and `[EXTRA_INSTRUCTION]` content in `table_rag_retrieve` results.
|
||||
- If results are truncated, explicitly tell the user total matches (`N+M`), displayed count (`N`), and omitted count (`M`).
|
||||
- Cite data sources using filenames from `file_ref_table`.
|
||||
|
||||
### 7. Citation Requirements for Retrieved Knowledge
|
||||
- When using knowledge from `rag_retrieve` or `table_rag_retrieve`, you MUST generate `<CITATION ... />` tags.
|
||||
- Follow the citation format returned by each tool.
|
||||
- Place citations immediately after the paragraph or bullet list that uses the knowledge.
|
||||
- Do NOT collect citations at the end.
|
||||
- Use 1-2 citations per paragraph or bullet list when possible.
|
||||
- If learned knowledge is used, include at least 1 `<CITATION ... />`.
|
||||
251
skills_autoload/rag-retrieve/mcp_common.py
Normal file
251
skills_autoload/rag-retrieve/mcp_common.py
Normal file
@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MCP服务器通用工具函数
|
||||
提供路径处理、文件验证、请求处理等公共功能
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import asyncio
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
import re
|
||||
|
||||
def get_allowed_directory():
|
||||
"""获取允许访问的目录"""
|
||||
# 优先使用命令行参数传入的dataset_dir
|
||||
if len(sys.argv) > 1:
|
||||
dataset_dir = sys.argv[1]
|
||||
return os.path.abspath(dataset_dir)
|
||||
|
||||
# 从环境变量读取项目数据目录
|
||||
project_dir = os.getenv("PROJECT_DATA_DIR", "./projects/data")
|
||||
return os.path.abspath(project_dir)
|
||||
|
||||
|
||||
def resolve_file_path(file_path: str, default_subfolder: str = "default") -> str:
|
||||
"""
|
||||
解析文件路径，支持 folder/document.txt 和 document.txt 两种格式
|
||||
|
||||
Args:
|
||||
file_path: 输入的文件路径
|
||||
default_subfolder: 当只传入文件名时使用的默认子文件夹名称
|
||||
|
||||
Returns:
|
||||
解析后的完整文件路径
|
||||
"""
|
||||
# 如果路径包含文件夹分隔符，直接使用
|
||||
if '/' in file_path or '\\' in file_path:
|
||||
clean_path = file_path.replace('\\', '/')
|
||||
|
||||
# 移除 projects/ 前缀（如果存在）
|
||||
if clean_path.startswith('projects/'):
|
||||
clean_path = clean_path[9:] # 移除 'projects/' 前缀
|
||||
elif clean_path.startswith('./projects/'):
|
||||
clean_path = clean_path[11:] # 移除 './projects/' 前缀
|
||||
else:
|
||||
# 如果只有文件名，添加默认子文件夹
|
||||
clean_path = f"{default_subfolder}/{file_path}"
|
||||
|
||||
# 获取允许的目录
|
||||
project_data_dir = get_allowed_directory()
|
||||
|
||||
# 尝试在项目目录中查找文件
|
||||
full_path = os.path.join(project_data_dir, clean_path.lstrip('./'))
|
||||
if os.path.exists(full_path):
|
||||
return full_path
|
||||
|
||||
# 如果直接路径不存在，尝试递归查找
|
||||
found = find_file_in_project(clean_path, project_data_dir)
|
||||
if found:
|
||||
return found
|
||||
|
||||
# 如果是纯文件名且在default子文件夹中不存在，尝试在根目录查找
|
||||
if '/' not in file_path and '\\' not in file_path:
|
||||
root_path = os.path.join(project_data_dir, file_path)
|
||||
if os.path.exists(root_path):
|
||||
return root_path
|
||||
|
||||
raise FileNotFoundError(f"File not found: {file_path} (searched in {project_data_dir})")
|
||||
|
||||
|
||||
def find_file_in_project(filename: str, project_dir: str) -> Optional[str]:
|
||||
"""在项目目录中递归查找文件"""
|
||||
# 如果filename包含路径，只搜索指定的路径
|
||||
if '/' in filename:
|
||||
parts = filename.split('/')
|
||||
target_file = parts[-1]
|
||||
search_dir = os.path.join(project_dir, *parts[:-1])
|
||||
|
||||
if os.path.exists(search_dir):
|
||||
target_path = os.path.join(search_dir, target_file)
|
||||
if os.path.exists(target_path):
|
||||
return target_path
|
||||
else:
|
||||
# 纯文件名，递归搜索整个项目目录
|
||||
for root, dirs, files in os.walk(project_dir):
|
||||
if filename in files:
|
||||
return os.path.join(root, filename)
|
||||
return None
|
||||
|
||||
|
||||
def load_tools_from_json(tools_file_name: str) -> List[Dict[str, Any]]:
|
||||
"""从 JSON 文件加载工具定义"""
|
||||
try:
|
||||
tools_file = os.path.join(os.path.dirname(__file__), tools_file_name)
|
||||
if os.path.exists(tools_file):
|
||||
with open(tools_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
# 如果 JSON 文件不存在，使用默认定义
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"Warning: Unable to load tool definition JSON file: {str(e)}")
|
||||
return []
|
||||
|
||||
|
||||
def create_error_response(request_id: Any, code: int, message: str) -> Dict[str, Any]:
|
||||
"""创建标准化的错误响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_success_response(request_id: Any, result: Any) -> Dict[str, Any]:
|
||||
"""创建标准化的成功响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": result
|
||||
}
|
||||
|
||||
|
||||
def create_initialize_response(request_id: Any, server_name: str, server_version: str = "1.0.0") -> Dict[str, Any]:
|
||||
"""创建标准化的初始化响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {
|
||||
"tools": {}
|
||||
},
|
||||
"serverInfo": {
|
||||
"name": server_name,
|
||||
"version": server_version
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_ping_response(request_id: Any) -> Dict[str, Any]:
|
||||
"""创建标准化的ping响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"pong": True
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def create_tools_list_response(request_id: Any, tools: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""创建标准化的工具列表响应"""
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"result": {
|
||||
"tools": tools
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def is_regex_pattern(pattern: str) -> bool:
|
||||
"""检测字符串是否为正则表达式模式"""
|
||||
# 检查 /pattern/ 格式
|
||||
if pattern.startswith('/') and pattern.endswith('/') and len(pattern) > 2:
|
||||
return True
|
||||
|
||||
# 检查 r"pattern" 或 r'pattern' 格式
|
||||
if pattern.startswith(('r"', "r'")) and pattern.endswith(('"', "'")) and len(pattern) > 3:
|
||||
return True
|
||||
|
||||
# 检查是否包含正则特殊字符
|
||||
regex_chars = {'*', '+', '?', '|', '(', ')', '[', ']', '{', '}', '^', '$', '\\', '.'}
|
||||
return any(char in pattern for char in regex_chars)
|
||||
|
||||
|
||||
def compile_pattern(pattern: str) -> Union[re.Pattern, str, None]:
|
||||
"""编译正则表达式模式，如果不是正则则返回原字符串"""
|
||||
if not is_regex_pattern(pattern):
|
||||
return pattern
|
||||
|
||||
try:
|
||||
# 处理 /pattern/ 格式
|
||||
if pattern.startswith('/') and pattern.endswith('/'):
|
||||
regex_body = pattern[1:-1]
|
||||
return re.compile(regex_body)
|
||||
|
||||
# 处理 r"pattern" 或 r'pattern' 格式
|
||||
if pattern.startswith(('r"', "r'")) and pattern.endswith(('"', "'")):
|
||||
regex_body = pattern[2:-1]
|
||||
return re.compile(regex_body)
|
||||
|
||||
# 直接编译包含正则字符的字符串
|
||||
return re.compile(pattern)
|
||||
except re.error as e:
|
||||
# 如果编译失败，返回None表示无效的正则
|
||||
print(f"Warning: Regular expression '{pattern}' compilation failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
async def handle_mcp_streaming(request_handler):
|
||||
"""处理MCP请求的标准主循环"""
|
||||
try:
|
||||
while True:
|
||||
# Read from stdin
|
||||
line = await asyncio.get_event_loop().run_in_executor(None, sys.stdin.readline)
|
||||
if not line:
|
||||
break
|
||||
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
request = json.loads(line)
|
||||
response = await request_handler(request)
|
||||
|
||||
# Write to stdout
|
||||
sys.stdout.write(json.dumps(response, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
except json.JSONDecodeError:
|
||||
error_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"error": {
|
||||
"code": -32700,
|
||||
"message": "Parse error"
|
||||
}
|
||||
}
|
||||
sys.stdout.write(json.dumps(error_response, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
except Exception as e:
|
||||
error_response = {
|
||||
"jsonrpc": "2.0",
|
||||
"error": {
|
||||
"code": -32603,
|
||||
"message": f"Internal error: {str(e)}"
|
||||
}
|
||||
}
|
||||
sys.stdout.write(json.dumps(error_response, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
@ -7,6 +7,7 @@ RAG检索MCP服务器
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from typing import Any, Dict, List
|
||||
@ -218,6 +219,12 @@ def table_rag_retrieve(query: str) -> Dict[str, Any]:
|
||||
|
||||
if "markdown" in response_data:
|
||||
markdown_content = response_data["markdown"]
|
||||
if re.search(r"^no excel files found", markdown_content, re.IGNORECASE):
|
||||
rag_result = rag_retrieve(query)
|
||||
content = rag_result.get("content", [])
|
||||
if content and content[0].get("type") == "text":
|
||||
content[0]["text"] = "No table_rag_retrieve results were found. The content below is the fallback result from rag_retrieve：\n\n" + content[0]["text"]
|
||||
return rag_result
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
35
skills_autoload/rag-retrieve/rag_retrieve_tools.json
Normal file
35
skills_autoload/rag-retrieve/rag_retrieve_tools.json
Normal file
@ -0,0 +1,35 @@
|
||||
[
|
||||
{
|
||||
"name": "rag_retrieve",
|
||||
"description": "Retrieve relevant documents from the knowledge base. Returns markdown results. Use this tool first only for clearly pure concept, definition, workflow, policy, or explanation questions without structured data needs. If the result is insufficient, try table_rag_retrieve before replying with no result.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Retrieval query content. Rewrite the query when needed to improve recall."
|
||||
},
|
||||
"top_k": {
|
||||
"type": "integer",
|
||||
"description": "Number of top results to retrieve. Choose dynamically based on retrieval breadth and coverage needs.",
|
||||
"default": 100
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "table_rag_retrieve",
|
||||
"description": "Retrieve relevant table data from Excel or spreadsheet files in the knowledge base. Returns markdown results. Use this tool first for structured data, lists, statistics, extraction, mixed questions, and unclear cases. If the result is insufficient, try rag_retrieve before replying with no result.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Retrieval query content for table data. Rewrite the query when needed to improve recall."
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
}
|
||||
}
|
||||
]
|
||||
48
skills_developing/rag-retrieve-cli/Retrieval_Policy.md
Normal file
48
skills_developing/rag-retrieve-cli/Retrieval_Policy.md
Normal file
@ -0,0 +1,48 @@
|
||||
## Retrieval Policy (Priority & Fallback)
|
||||
|
||||
### 1. Retrieval Source Priority
|
||||
- If earlier context does not explicitly specify a knowledge retrieval priority, the default order is: skill-enabled knowledge retrieval tools > `rag_retrieve` / `table_rag_retrieve` > local filesystem retrieval (including `datasets/` and any file browsing/search tools).
|
||||
- Follow this `Retrieval Policy (Priority & Fallback)` section for retrieval source selection, tool selection order, query rewrite, `top_k`, result handling, fallback, and citation requirements.
|
||||
- The local filesystem is the lowest-priority source. Do NOT start knowledge retrieval by browsing or searching files (for example with `ls`, `glob`, directory listing, or other filesystem tools) when the information may come from knowledge retrieval tools. Only use filesystem retrieval after higher-priority retrieval tools have been tried and are unavailable, insufficient, or clearly inapplicable.
|
||||
|
||||
### 2. Tool Selection
|
||||
- When knowledge retrieval is needed and no higher-priority skill-enabled retrieval tool is specified or available, you MUST start with `rag_retrieve` or `table_rag_retrieve` based on the question type. Do NOT answer from model knowledge before trying the appropriate retrieval tool.
|
||||
- Use `table_rag_retrieve` first for values, prices, quantities, inventory, specifications, rankings, comparisons, summaries, extraction, lists, tables, person / project / product name lookup, historical coverage, mixed questions, or any unclear case.
|
||||
- Use `rag_retrieve` first only for clearly pure concept / definition / workflow / policy / explanation questions that do not need structured data.
|
||||
|
||||
### 3. Query Preparation
|
||||
- Do NOT pass the user's raw question directly unless it already fits retrieval needs well.
|
||||
- Rewrite the query to improve recall: extract the core entity, time scope, attributes, and intent.
|
||||
- Add meaningful variants such as synonyms, aliases, abbreviations, related titles, historical names, and category terms.
|
||||
- Expand enumeration-style, historical, roster, timeline, overview, archive, extraction, and list-style queries more aggressively.
|
||||
- Preserve the original meaning and do not introduce unrelated topics. Use both the original query and rewritten variants whenever possible.
|
||||
|
||||
### 4. Retrieval Breadth (`top_k`)
|
||||
- `top_k` applies to `rag_retrieve`. Use the smallest sufficient `top_k` and expand only when coverage is insufficient.
|
||||
- Use `30` for simple fact lookup about one specific thing.
|
||||
- Use `50` for moderate synthesis, comparison, summarization, or disambiguation.
|
||||
- Use `100` for broad-recall queries needing high coverage, such as comprehensive analysis, scattered knowledge, multiple entities or periods, list / catalog / timeline / roster / overview requests, or all items / historical succession / many records.
|
||||
- Raise `top_k` when query rewrite produces many useful keyword branches or when results are too few, repetitive, incomplete, sparse, or too narrow in coverage. Do not raise `top_k` just because the query is longer.
|
||||
- Expansion sequence: `30 -> 50 -> 100`. If uncertain, prefer `100`.
|
||||
|
||||
### 5. Result Evaluation
|
||||
- Treat the result as insufficient when it is empty, starts with `Error:`, says `no excel files found`, is off-topic, does not match the user's core entity / scope, or clearly contains no usable evidence.
|
||||
- Treat the result as insufficient when it only covers part of the user's request, or when the user asked for a complete list, historical coverage, comparison, or mixed data + explanation but the result is only partial or truncated.
|
||||
|
||||
### 6. Fallback and Sequential Retry
|
||||
- If the first retrieval tool returns empty results, errors, clearly irrelevant content, or only partial coverage of the user's request, you MUST try the other retrieval tool before replying to the user.
|
||||
- If the table result is empty, continue with `rag_retrieve` before concluding that no relevant data exists.
|
||||
- You may say that no relevant information was found only after both `rag_retrieve` and `table_rag_retrieve` have been tried and still do not provide enough evidence to answer.
|
||||
|
||||
### 7. Table Result Handling
|
||||
- When processing `table_rag_retrieve` results, follow all instructions in `[INSTRUCTION]` and `[EXTRA_INSTRUCTION]` sections of the response.
|
||||
- If Query result hint indicates truncation (for example, `Only the first N rows are included; the remaining M rows were omitted`), you MUST explicitly tell the user the total matches (`N+M`), displayed count (`N`), and omitted count (`M`).
|
||||
- Cite data sources using file names from `file_ref_table` in the response.
|
||||
|
||||
### 8. Citation Requirements for Retrieved Knowledge
|
||||
- When your answer uses learned knowledge from `rag_retrieve` or `table_rag_retrieve`, you MUST generate `<CITATION ... />` tags.
|
||||
- Follow the specific citation format instructions returned by each tool.
|
||||
- Citations MUST appear IMMEDIATELY AFTER the paragraph or bullet list that uses the knowledge.
|
||||
- NEVER collect all citations and place them at the end of your response.
|
||||
- Limit to 1-2 citations per paragraph or bullet list, combining related facts under one citation when possible.
|
||||
- If your answer uses learned knowledge, you MUST generate at least 1 `<CITATION ... />` in the response.
|
||||
197
skills_developing/rag-retrieve-cli/SKILL.md
Normal file
197
skills_developing/rag-retrieve-cli/SKILL.md
Normal file
@ -0,0 +1,197 @@
|
||||
---
|
||||
name: rag-retrieve
|
||||
description: RAG retrieval skill for querying and retrieving relevant documents from knowledge base. Use this skill when users need to search documentation, retrieve knowledge base articles, or get context from a vector database. Supports semantic search with configurable top-k results.
|
||||
---
|
||||
|
||||
# RAG Retrieve
|
||||
|
||||
## Skill Structure
|
||||
|
||||
This is a **self-contained skill package** that can be distributed independently. The skill includes its own scripts and configuration:
|
||||
|
||||
```
|
||||
rag-retrieve/
|
||||
├── SKILL.md # Core instruction file (this file)
|
||||
├── skill.yaml # Skill metadata
|
||||
├── scripts/ # Executable scripts
|
||||
│ └── rag_retrieve.py # Main RAG retrieval script
|
||||
```
|
||||
|
||||
## Overview
|
||||
|
||||
Query and retrieve relevant documents from a RAG (Retrieval-Augmented Generation) knowledge base using vector search. This skill provides semantic search capabilities with support for multiple bot instances and configurable result limits.
|
||||
|
||||
## Required Parameters
|
||||
|
||||
Before executing any retrieval, you MUST confirm the following required parameters with the user if they are not explicitly provided:
|
||||
|
||||
| Parameter | Description | Type |
|
||||
|-----------|-------------|------|
|
||||
| **query** | Search query content | string |
|
||||
|
||||
### Optional Parameters
|
||||
|
||||
| Parameter | Description | Type | Default |
|
||||
|-----------|-------------|------|---------|
|
||||
| **top_k** | Maximum number of results | integer | 100 |
|
||||
|
||||
### Confirmation Template
|
||||
|
||||
When the required parameter is missing, ask the user:
|
||||
|
||||
```
|
||||
I need some information to perform the RAG retrieval:
|
||||
|
||||
1. Query: What would you like to search for?
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
Use the `scripts/rag_retrieve.py` script to execute RAG queries:
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "your search query"
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Query
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "How to configure authentication?"
|
||||
```
|
||||
|
||||
### Search with Specific Top-K
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "API error handling" --top-k 50
|
||||
```
|
||||
|
||||
### Common Use Cases
|
||||
|
||||
**Scenario 1: Documentation Search**
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "deployment guide"
|
||||
```
|
||||
|
||||
**Scenario 2: Troubleshooting**
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "connection timeout error"
|
||||
```
|
||||
|
||||
**Scenario 3: Feature Information**
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "enterprise pricing plans"
|
||||
```
|
||||
|
||||
## Script Usage
|
||||
|
||||
### rag_retrieve.py
|
||||
|
||||
Main script for executing RAG retrieval queries.
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
|
||||
| Option | Required | Description | Default |
|
||||
|--------|----------|-------------|---------|
|
||||
| `--query`, `-q` | Yes | Search query content | - |
|
||||
| `--top-k`, `-k` | No | Maximum number of results | 100 |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
# Basic query
|
||||
scripts/rag_retrieve.py --query "authentication setup"
|
||||
|
||||
# Custom top-k
|
||||
scripts/rag_retrieve.py --query "API reference" --top-k 20
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Research Mode: Comprehensive Search
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "machine learning algorithms" --top-k 100
|
||||
```
|
||||
|
||||
### Quick Answer Mode: Focused Search
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "password reset" --top-k 10
|
||||
```
|
||||
|
||||
### Comparison Mode: Multiple Queries
|
||||
|
||||
```bash
|
||||
# Search for related topics
|
||||
scripts/rag_retrieve.py --query "REST API" --top-k 30
|
||||
scripts/rag_retrieve.py --query "GraphQL API" --top-k 30
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
### scripts/rag_retrieve.py
|
||||
|
||||
Executable Python script for RAG retrieval. Handles:
|
||||
- HTTP requests to RAG API
|
||||
- Authentication token generation
|
||||
- Configuration file loading
|
||||
- Error handling and reporting
|
||||
- Markdown response parsing
|
||||
|
||||
The script can be executed directly without loading into context.
|
||||
|
||||
|
||||
## Retrieval Policy (Priority & Fallback)
|
||||
|
||||
### 1. Retrieval Source Priority
|
||||
- If earlier context does not explicitly specify a knowledge retrieval priority, the default order is: skill-enabled knowledge retrieval tools > `rag_retrieve` / `table_rag_retrieve` > local filesystem retrieval (including `datasets/` and any file browsing/search tools).
|
||||
- Follow this `Retrieval Policy (Priority & Fallback)` section for retrieval source selection, tool selection order, query rewrite, `top_k`, result handling, fallback, and citation requirements.
|
||||
- The local filesystem is the lowest-priority source. Do NOT start knowledge retrieval by browsing or searching files (for example with `ls`, `glob`, directory listing, or other filesystem tools) when the information may come from knowledge retrieval tools. Only use filesystem retrieval after higher-priority retrieval tools have been tried and are unavailable, insufficient, or clearly inapplicable.
|
||||
|
||||
### 2. Tool Selection
|
||||
- When knowledge retrieval is needed and no higher-priority skill-enabled retrieval tool is specified or available, you MUST start with `rag_retrieve` or `table_rag_retrieve` based on the question type. Do NOT answer from model knowledge before trying the appropriate retrieval tool.
|
||||
- Use `table_rag_retrieve` first for values, prices, quantities, inventory, specifications, rankings, comparisons, summaries, extraction, lists, tables, person / project / product name lookup, historical coverage, mixed questions, or any unclear case.
|
||||
- Use `rag_retrieve` first only for clearly pure concept / definition / workflow / policy / explanation questions that do not need structured data.
|
||||
|
||||
### 3. Query Preparation
|
||||
- Do NOT pass the user's raw question directly unless it already fits retrieval needs well.
|
||||
- Rewrite the query to improve recall: extract the core entity, time scope, attributes, and intent.
|
||||
- Add meaningful variants such as synonyms, aliases, abbreviations, related titles, historical names, and category terms.
|
||||
- Expand enumeration-style, historical, roster, timeline, overview, archive, extraction, and list-style queries more aggressively.
|
||||
- Preserve the original meaning and do not introduce unrelated topics. Use both the original query and rewritten variants whenever possible.
|
||||
|
||||
### 4. Retrieval Breadth (`top_k`)
|
||||
- `top_k` applies to `rag_retrieve`. Use the smallest sufficient `top_k` and expand only when coverage is insufficient.
|
||||
- Use `30` for simple fact lookup about one specific thing.
|
||||
- Use `50` for moderate synthesis, comparison, summarization, or disambiguation.
|
||||
- Use `100` for broad-recall queries needing high coverage, such as comprehensive analysis, scattered knowledge, multiple entities or periods, list / catalog / timeline / roster / overview requests, or all items / historical succession / many records.
|
||||
- Raise `top_k` when query rewrite produces many useful keyword branches or when results are too few, repetitive, incomplete, sparse, or too narrow in coverage. Do not raise `top_k` just because the query is longer.
|
||||
- Expansion sequence: `30 -> 50 -> 100`. If uncertain, prefer `100`.
|
||||
|
||||
### 5. Result Evaluation
|
||||
- Treat the result as insufficient when it is empty, starts with `Error:`, says `no excel files found`, is off-topic, does not match the user's core entity / scope, or clearly contains no usable evidence.
|
||||
- Treat the result as insufficient when it only covers part of the user's request, or when the user asked for a complete list, historical coverage, comparison, or mixed data + explanation but the result is only partial or truncated.
|
||||
|
||||
### 6. Fallback and Sequential Retry
|
||||
- If the first retrieval tool returns empty results, errors, clearly irrelevant content, or only partial coverage of the user's request, you MUST try the other retrieval tool before replying to the user.
|
||||
- If the table result is empty, continue with `rag_retrieve` before concluding that no relevant data exists.
|
||||
- You may say that no relevant information was found only after both `rag_retrieve` and `table_rag_retrieve` have been tried and still do not provide enough evidence to answer.
|
||||
|
||||
### 7. Table Result Handling
|
||||
- When processing `table_rag_retrieve` results, follow all instructions in `[INSTRUCTION]` and `[EXTRA_INSTRUCTION]` sections of the response.
|
||||
- If Query result hint indicates truncation (for example, `Only the first N rows are included; the remaining M rows were omitted`), you MUST explicitly tell the user the total matches (`N+M`), displayed count (`N`), and omitted count (`M`).
|
||||
- Cite data sources using file names from `file_ref_table` in the response.
|
||||
|
||||
### 8. Citation Requirements for Retrieved Knowledge
|
||||
- When your answer uses learned knowledge from `rag_retrieve` or `table_rag_retrieve`, you MUST generate `<CITATION ... />` tags.
|
||||
- Follow the specific citation format instructions returned by each tool.
|
||||
- Citations MUST appear IMMEDIATELY AFTER the paragraph or bullet list that uses the knowledge.
|
||||
- NEVER collect all citations and place them at the end of your response.
|
||||
- Limit to 1-2 citations per paragraph or bullet list, combining related facts under one citation when possible.
|
||||
- If your answer uses learned knowledge, you MUST generate at least 1 `<CITATION ... />` in the response.
|
||||
@ -1,147 +0,0 @@
|
||||
---
|
||||
name: rag-retrieve
|
||||
description: RAG retrieval skill for querying and retrieving relevant documents from knowledge base. Use this skill when users need to search documentation, retrieve knowledge base articles, or get context from a vector database. Supports semantic search with configurable top-k results.
|
||||
---
|
||||
|
||||
# RAG Retrieve
|
||||
|
||||
## Skill Structure
|
||||
|
||||
This is a **self-contained skill package** that can be distributed independently. The skill includes its own scripts and configuration:
|
||||
|
||||
```
|
||||
rag-retrieve/
|
||||
├── SKILL.md # Core instruction file (this file)
|
||||
├── skill.yaml # Skill metadata
|
||||
├── scripts/ # Executable scripts
|
||||
│ └── rag_retrieve.py # Main RAG retrieval script
|
||||
```
|
||||
|
||||
## Overview
|
||||
|
||||
Query and retrieve relevant documents from a RAG (Retrieval-Augmented Generation) knowledge base using vector search. This skill provides semantic search capabilities with support for multiple bot instances and configurable result limits.
|
||||
|
||||
## Required Parameters
|
||||
|
||||
Before executing any retrieval, you MUST confirm the following required parameters with the user if they are not explicitly provided:
|
||||
|
||||
| Parameter | Description | Type |
|
||||
|-----------|-------------|------|
|
||||
| **query** | Search query content | string |
|
||||
|
||||
### Optional Parameters
|
||||
|
||||
| Parameter | Description | Type | Default |
|
||||
|-----------|-------------|------|---------|
|
||||
| **top_k** | Maximum number of results | integer | 100 |
|
||||
|
||||
### Confirmation Template
|
||||
|
||||
When the required parameter is missing, ask the user:
|
||||
|
||||
```
|
||||
I need some information to perform the RAG retrieval:
|
||||
|
||||
1. Query: What would you like to search for?
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
Use the `scripts/rag_retrieve.py` script to execute RAG queries:
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "your search query"
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Query
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "How to configure authentication?"
|
||||
```
|
||||
|
||||
### Search with Specific Top-K
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "API error handling" --top-k 50
|
||||
```
|
||||
|
||||
### Common Use Cases
|
||||
|
||||
**Scenario 1: Documentation Search**
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "deployment guide"
|
||||
```
|
||||
|
||||
**Scenario 2: Troubleshooting**
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "connection timeout error"
|
||||
```
|
||||
|
||||
**Scenario 3: Feature Information**
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "enterprise pricing plans"
|
||||
```
|
||||
|
||||
## Script Usage
|
||||
|
||||
### rag_retrieve.py
|
||||
|
||||
Main script for executing RAG retrieval queries.
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py [OPTIONS]
|
||||
```
|
||||
|
||||
**Options:**
|
||||
|
||||
| Option | Required | Description | Default |
|
||||
|--------|----------|-------------|---------|
|
||||
| `--query`, `-q` | Yes | Search query content | - |
|
||||
| `--top-k`, `-k` | No | Maximum number of results | 100 |
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
# Basic query
|
||||
scripts/rag_retrieve.py --query "authentication setup"
|
||||
|
||||
# Custom top-k
|
||||
scripts/rag_retrieve.py --query "API reference" --top-k 20
|
||||
```
|
||||
|
||||
## Common Workflows
|
||||
|
||||
### Research Mode: Comprehensive Search
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "machine learning algorithms" --top-k 100
|
||||
```
|
||||
|
||||
### Quick Answer Mode: Focused Search
|
||||
|
||||
```bash
|
||||
scripts/rag_retrieve.py --query "password reset" --top-k 10
|
||||
```
|
||||
|
||||
### Comparison Mode: Multiple Queries
|
||||
|
||||
```bash
|
||||
# Search for related topics
|
||||
scripts/rag_retrieve.py --query "REST API" --top-k 30
|
||||
scripts/rag_retrieve.py --query "GraphQL API" --top-k 30
|
||||
```
|
||||
|
||||
## Resources
|
||||
|
||||
### scripts/rag_retrieve.py
|
||||
|
||||
Executable Python script for RAG retrieval. Handles:
|
||||
- HTTP requests to RAG API
|
||||
- Authentication token generation
|
||||
- Configuration file loading
|
||||
- Error handling and reporting
|
||||
- Markdown response parsing
|
||||
|
||||
The script can be executed directly without loading into context.
|
||||
@ -7,6 +7,7 @@ import os
|
||||
import shutil
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
from datetime import datetime
|
||||
@ -320,6 +321,12 @@ def create_robot_project(dataset_ids: List[str], bot_id: str, force_rebuild: boo
|
||||
str: 机器人项目目录路径
|
||||
"""
|
||||
|
||||
skills = list(skills or [])
|
||||
has_rag_retrieve = any(re.search(r"rag-retrieve", skill) for skill in skills)
|
||||
if dataset_ids and not has_rag_retrieve:
|
||||
skills.append("@skills_autoload/rag-retrieve")
|
||||
logger.info("Auto loaded skill '@skills_autoload/rag-retrieve' because dataset_ids is not empty")
|
||||
|
||||
logger.info(f"Ensuring robot project exists: {bot_id}, skills: {skills}")
|
||||
|
||||
# 创建机器人目录结构（如果不存在）
|
||||
@ -375,59 +382,72 @@ def _extract_skills_to_robot(bot_id: str, skills: List[str], project_path: Path)
|
||||
- 如果是简单名称（如 "rag-retrieve"），从以下目录按优先级顺序查找：
|
||||
1. projects/uploads/{bot_id}/skills/
|
||||
2. skills/
|
||||
- 如果是以 @ 开头的仓库相对路径（如 "@skills_autoload/rag-retrieve"），则从仓库根目录直接解析
|
||||
|
||||
搜索目录优先级：先搜索 projects/uploads/{bot_id}/skills/，再搜索 skills/
|
||||
|
||||
Args:
|
||||
bot_id: 机器人 ID
|
||||
skills: 技能文件名列表（如 ["rag-retrieve", "projects/uploads/{bot_id}/skills/rag-retrieve"]）
|
||||
skills: 技能文件名列表（如 ["rag-retrieve", "@skills_autoload/rag-retrieve", "projects/uploads/{bot_id}/skills/rag-retrieve"]）
|
||||
project_path: 项目路径
|
||||
"""
|
||||
import zipfile
|
||||
|
||||
# skills 源目录（按优先级顺序）
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
skills_source_dirs = [
|
||||
project_path / "uploads" / bot_id / "skills",
|
||||
Path("skills"),
|
||||
repo_root / "skills",
|
||||
]
|
||||
skills_target_dir = project_path / "robot" / bot_id / "skills"
|
||||
skills_target_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"Copying skills to {skills_target_dir}")
|
||||
|
||||
managed_skill_names = set()
|
||||
for base_dir in skills_source_dirs:
|
||||
if not base_dir.exists():
|
||||
continue
|
||||
for item in base_dir.iterdir():
|
||||
if item.is_dir():
|
||||
managed_skill_names.add(item.name)
|
||||
|
||||
# 清理不在列表中的多余 skill 文件夹
|
||||
expected_skill_names = {os.path.basename(skill) for skill in skills}
|
||||
expected_skill_names = {Path(skill.lstrip("@")).name for skill in skills}
|
||||
if skills_target_dir.exists():
|
||||
for item in skills_target_dir.iterdir():
|
||||
if item.is_dir() and item.name not in expected_skill_names:
|
||||
logger.info(f" Removing stale skill directory: {item}")
|
||||
if not item.is_dir() or item.name in expected_skill_names:
|
||||
continue
|
||||
if item.name in managed_skill_names:
|
||||
logger.info(f" Removing managed stale skill directory: {item}")
|
||||
shutil.rmtree(item)
|
||||
else:
|
||||
logger.info(f" Keeping unmanaged skill directory: {item}")
|
||||
|
||||
for skill in skills:
|
||||
target_dir = skills_target_dir / os.path.basename(skill)
|
||||
|
||||
# 如果目标目录已存在，跳过复制
|
||||
if target_dir.exists():
|
||||
logger.info(f" Skill '{skill}' already exists in {target_dir}, skipping")
|
||||
continue
|
||||
skill_name = Path(skill.lstrip("@")).name
|
||||
target_dir = skills_target_dir / skill_name
|
||||
|
||||
source_dir = None
|
||||
|
||||
# 简单名称：按优先级顺序在多个目录中查找
|
||||
for base_dir in skills_source_dirs:
|
||||
candidate_dir = base_dir / skill
|
||||
if skill.startswith("@"):
|
||||
candidate_dir = repo_root / skill.lstrip("@")
|
||||
if candidate_dir.exists():
|
||||
source_dir = candidate_dir
|
||||
logger.info(f" Found skill '{skill}' in {base_dir}")
|
||||
break
|
||||
logger.info(f" Found skill '{skill}' at {candidate_dir}")
|
||||
|
||||
# 简单名称：按优先级顺序在多个目录中查找
|
||||
if source_dir is None:
|
||||
for base_dir in skills_source_dirs:
|
||||
candidate_dir = base_dir / skill
|
||||
if candidate_dir.exists():
|
||||
source_dir = candidate_dir
|
||||
logger.info(f" Found skill '{skill}' in {base_dir}")
|
||||
break
|
||||
|
||||
if source_dir is None:
|
||||
logger.warning(f" Skill directory '{skill}' not found in any source directory: {[str(d) for d in skills_source_dirs]}")
|
||||
continue
|
||||
|
||||
target_dir = skills_target_dir / os.path.basename(skill)
|
||||
|
||||
try:
|
||||
shutil.copytree(source_dir, target_dir)
|
||||
logger.info(f" Copied: {source_dir} -> {target_dir}")
|
||||
shutil.copytree(source_dir, target_dir, dirs_exist_ok=True)
|
||||
logger.info(f" Synced: {source_dir} -> {target_dir}")
|
||||
except Exception as e:
|
||||
logger.error(f" Failed to copy {source_dir}: {e}")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user