diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 51c70dfcf..e7e4754f5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,6 +22,7 @@ # 引擎 / 策略 / 经纪商 / 数据源（R2） /backtrader/cerebro.py @cloudQuant +/backtrader/_cerebro/ @cloudQuant /backtrader/strategy.py @cloudQuant /backtrader/broker.py @cloudQuant /backtrader/brokers/ @cloudQuant diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index f57d965d6..0417e96a2 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -24,7 +24,7 @@ body: attributes: label: 环境 description: Python 版本、操作系统、backtrader 版本、安装方式 - placeholder: "Python 3.11, macOS, backtrader 1.3.0, pip install -e ." + placeholder: "Python 3.11, macOS, backtrader 1.4.0, pip install -e ." validations: required: true - type: textarea diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..53673d219 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,56 @@ +name: CodeQL Security Analysis + +on: + push: + branches: [dev, development, master] + pull_request: + branches: [dev, development, master] + schedule: + - cron: '30 5 * * 1' + +permissions: + contents: read + security-events: write + +jobs: + analyze: + name: Analyze Python + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + # Quality findings are owned by ruff/mypy/bandit; CodeQL focuses on + # security here. The inline config narrows the scan to released + # product code (backtrader/) and disables queries that conflict with + # the framework design: direct bt.Strategy subclasses deliberately + # never call super().__init__() (Strategy.__new__ already performs + # base initialization), so py/missing-super-init is a pure false + # positive across example strategies. + queries: security-extended + config: | + paths-ignore: + - docs + - examples + - tests + - tools + - studies + - scripts + - .joyincode + - conftest.py + disable: + queries: + - py/missing-super-init + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@v3 + with: + category: '/language:python' diff --git a/.github/workflows/docs-auto-build.yml b/.github/workflows/docs-auto-build.yml index fc8ec87bd..6ef63ccee 100644 --- a/.github/workflows/docs-auto-build.yml +++ b/.github/workflows/docs-auto-build.yml @@ -5,7 +5,7 @@ name: Documentation CI Checks on: pull_request: - branches: [development, master] + branches: [dev, development, master] paths: - 'docs/**' - 'backtrader/**/*.py' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c2399a7b4..125dbcc8a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -124,7 +124,10 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest needs: build - if: github.event_name == 'push' && (github.ref == 'refs/heads/development' || github.ref == 'refs/heads/master') + # The existing github-pages environment authorizes only development. + # Master releases still build both languages and retain the Pages artifact; + # publishing a different branch requires a separate environment-policy change. + if: github.event_name == 'push' && github.ref == 'refs/heads/development' steps: - name: Deploy to GitHub Pages id: deployment diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 91d37b62b..3fff8bcd1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,7 +54,7 @@ jobs: - name: Install linting tools run: | python -m pip install --upgrade pip - pip install ruff 'black==26.1.0' isort 'mypy==1.16.1' bandit pip-audit + pip install ruff 'black==26.1.0' 'isort==9.0.1' 'mypy==1.16.1' bandit pip-audit - name: Run ruff run: ruff check backtrader/ @@ -81,7 +81,8 @@ jobs: # mypy==1.16.1 keeps the Python 3.8 target available; the current # gate is fully clean and should fail on any new type error. MYPY_THRESHOLD=0 - mypy backtrader --config-file=pyproject.toml | tee mypy-report.txt || true + set -o pipefail + mypy backtrader --config-file=pyproject.toml | tee mypy-report.txt count="$(grep -cE 'error:' mypy-report.txt || true)" echo "mypy error count: ${count} (gate threshold ${MYPY_THRESHOLD})" if [ "${count}" -gt "${MYPY_THRESHOLD}" ]; then @@ -229,12 +230,87 @@ jobs: # Choose tier by event: pull_request => fast gate; everything else => full suite. if [ "${EVENT_NAME}" = "pull_request" ]; then echo "=== PR fast gate: pytest -m 'not slow' ===" - pytest tests/ -m "not slow" -n auto --tb=short --timeout=300 -q + pytest tests/ -m "not slow and not performance" -n auto --tb=short --timeout=300 -q else echo "=== Full suite: pytest tests/ ===" - pytest tests/ -n auto --tb=short --timeout=300 -q + pytest tests/ -m "not performance" -n auto --tb=short --timeout=300 -q fi + sdk: + name: Optional SDK Contracts + runs-on: ubuntu-latest + needs: lint + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + - name: Install core and pinned SDK acceptance dependencies + run: python -m pip install -e ".[dev]" -r requirements-ci-sdk.txt + - name: Require SDK adapters and run complete functional suite + env: + PYTEST_ADDOPTS: '' + run: | + python -c "import bt_api_py, bt_api_base, bt_api_binance, bt_api_okx, bt_api_ctp, spdlog; print('All required SDK adapters present')" + # ``-n auto`` varies with runner CPU exposure. Eight workers are + # the release-validated SDK-contract profile and keep this job bounded. + python -m pytest tests -m 'not performance' -n 8 --tb=short --timeout=300 -q + + performance: + name: Serial Performance + runs-on: ubuntu-latest + needs: lint + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + - name: Install dependencies + run: python -m pip install -e ".[dev]" -r requirements-ci-sdk.txt + - name: Run serial performance and isolated RSS contracts + env: + PYTEST_ADDOPTS: '' + run: make test-performance BT_CONDA_PYTHON=python + + wheel-consumer: + name: Wheel Consumer + runs-on: ubuntu-latest + needs: lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + - name: Build and install isolated release artifacts + shell: bash + run: | + python -m pip install --upgrade pip setuptools wheel build twine + python -m pip install . + python -m build --outdir "$RUNNER_TEMP/dist" + python -m twine check "$RUNNER_TEMP"/dist/* + wheel_path=$(find "$RUNNER_TEMP/dist" -name '*.whl' -print -quit) + test -n "$wheel_path" + python -m pip install --no-deps --target "$RUNNER_TEMP/installed" "$wheel_path" + - name: Verify source identity and run consumer trades outside checkout + shell: bash + run: | + version=$(python -c "import runpy; print(runpy.run_path('backtrader/version.py')['__version__'])") + wheel_path=$(find "$RUNNER_TEMP/dist" -name '*.whl' -print -quit) + cd "$RUNNER_TEMP" + PYTHONPATH="$RUNNER_TEMP/installed" python "$GITHUB_WORKSPACE/scripts/ci/verify_release_wheel.py" \ + --wheel "$wheel_path" --source-root "$GITHUB_WORKSPACE" \ + --install-root "$RUNNER_TEMP/installed" --version "$version" \ + --output "$RUNNER_TEMP/wheel-consumer.json" + - uses: actions/upload-artifact@v4 + with: + name: wheel-consumer-evidence + path: ${{ runner.temp }}/wheel-consumer.json + coverage: name: Coverage (non-strategy subset, non-blocking floor) runs-on: ubuntu-latest @@ -269,13 +345,13 @@ jobs: test-summary: name: Test Summary runs-on: ubuntu-latest - needs: test + needs: [test, sdk, performance, wheel-consumer] if: always() steps: - name: Check test results run: | - if [ "${{ needs.test.result }}" == "success" ]; then - echo "All matrix tests passed." + if [ "${{ needs.test.result }}" == "success" ] && [ "${{ needs.sdk.result }}" == "success" ] && [ "${{ needs.performance.result }}" == "success" ] && [ "${{ needs.wheel-consumer.result }}" == "success" ]; then + echo "All matrix tests, serial performance contracts and wheel consumers passed." else echo "At least one matrix test job failed" exit 1 @@ -346,7 +422,7 @@ jobs: - name: Run development R2/R3 strategy regression gate if: github.base_ref == 'development' shell: bash - run: make test-strategies + run: make test-strategies BT_CONDA_PYTHON=python - name: Run original-baseline hotfix gate if: github.base_ref == 'master' diff --git a/.gitignore b/.gitignore index 6310946a2..2ee6a4e03 100644 --- a/.gitignore +++ b/.gitignore @@ -162,10 +162,10 @@ backtrader_remove_metaprogramming_report_timing.json performance_comparison.json backtrader-master/*htmlcov/ docs/_internal/opts/requirements/迭代3-宏源期货完成穿透式认证/ -examples/live_certification/hongyuan_penetration/reports/ +examples/007_ctp/live_certification/hongyuan_penetration/reports/ examples/003_hft_notebook_examples/data/ -examples/live_certification/simnow_penetration/reports/ -examples/live_certification/hongyuan_penetration/*.docx +examples/007_ctp/live_certification/simnow_penetration/reports/ +examples/007_ctp/live_certification/hongyuan_penetration/*.docx # Generated visualizations and reports from runnable examples /examples/output/ diff --git a/.joyincode/joyin-project-skills.json b/.joyincode/joyin-project-skills.json index e4c7f9f78..c826b534d 100644 --- a/.joyincode/joyin-project-skills.json +++ b/.joyincode/joyin-project-skills.json @@ -2,11 +2,12 @@ "projectId": "0", "managedEntries": [ "demo", + "未经授权禁止动已有代码", "specx", "opsx", - "devwiki", "jcdb", "markitdown", + "jccb", "grill-with-docs" ] } diff --git a/.joyincode/skills/devwiki/SKILL.md b/.joyincode/skills/devwiki/SKILL.md deleted file mode 100644 index 76c02bd64..000000000 --- a/.joyincode/skills/devwiki/SKILL.md +++ /dev/null @@ -1,93 +0,0 @@ ---- -name: devwiki -description: Use when the user asks about JoyinCode development workflow, how to use AI coding in this project, or mentions any pipeline step (markitdown Word-to-MD conversion, grill-with-docs requirement analysis, opsx proposal workflow). ---- - -# 标准开发流程 - -## 概述 -JoyinCode 推荐采用“转换 → 拷问 → 提案 → 实施 → 归档”五步法。每一步必须依次执行，不可跳跃。 - -## 开发流程 - -### 步骤 1：需求文档转换（markitdown） -**目的**：将非 Markdown 格式的需求文档（如 Word、PDF）转为纯文本 Markdown，便于后续 AI 读取和拷问 - -**操作**： -1. 在对话中上传需求文档（.docx/.pdf 等） -2. 输入指令：`转为md`（或 `markitdown`） -3. 工具自动将文档内容提取为 Markdown 格式，并保存至 `docs/` 目录下，文件名与原始文档相同（后缀改为 .md） - -**示例**：选择需求word文档并输入 `转为md` - -> 📌 转换后的文档仅包含文字内容，图片、表格等复杂元素会被忽略或转为文字描述。若原文档包含关键图表，建议在拷问步骤中人工补充说明 - -### 步骤 2：需求拷问（grill-with-docs） -**目的**：对转换后的需求 Markdown 进行多轮问答，挖掘隐含需求、识别歧义、补全缺失细节，确保需求可执行 - -**操作**： -1. 在对话中选中刚生成的 .md 文件（或指定文件路径） -2. 输入指令：`/grill-with-docs 开发[功能名称]`（例如 `/grill-with-docs 开发用户登录模块`） -3. 系统会基于需求内容主动提问（如异常流程、边界条件、性能指标等），开发者逐一回答 -4. 拷问完成后，需求文档中所有模糊点应已澄清，并获得一份“拷问纪要” - -**示例**：选择需求 md 文档，输入 `/grill-with-docs 开发xxx` - -> ⚠️ 关键约束：OpenSpec 无法直接引用拷问过程的成果物，因此 拷问与下一步“发起提案”必须在同一个对话会话中连续进行，以共享上下文。若中途切换对话，拷问成果将丢失 - -### 步骤 3：发起提案（opsx propose） -**目的**：基于拷问后的需求，生成 OpenSpec 规格变更提案，包括影响范围、变更清单、验收标准等 - -**操作**： -1. 确保当前对话仍包含步骤 2 的拷问上下文 -2. 选中需求 .md 文件（或直接引用） -3. 输入指令：`/opsx 发起提案` -4. OpenSpec 自动分析需求与现有规格，生成提案文档（位于 openspec/changes//） -5. 开发者务必人工审核生成的提案内容，确认： - - 变更描述准确 - - 影响模块覆盖完整 - - 验收条件可测 - - 与拷问结论一致 - -**示例**：选择需求 md 文档，输入 `/opsx 发起提案` - -> 💡 若提案不完善，可多次调用 `/opsx propose` 进行调整，或手动编辑提案文件后重新运行 - -### 步骤 4：实施提案（opsx apply） -**目的**：按照已审核通过的提案进行代码开发和测试 - -**操作**： -1. 确认提案内容已完全符合开发要求（可再次运行拷问或与团队评审） -2. 输入指令：`/opsx 实施提案`（或 `/opsx apply`） -3. OpenSpec 会根据提案生成任务清单，并引导开发者按任务顺序编码 -4. 完成所有任务后，运行项目的单元测试、集成测试，确保验收条件全部通过 - -**示例**：输入 `/opsx 实施提案` - -### 步骤 5：归档提案（opsx archive） -**目的**：开发完成且测试通过后，将提案状态标记为“已完成”，并归档规格变更，更新主规格文档 - -**操作**： -1. 确认代码已合并至目标分支，且所有测试通过 -2. 输入指令：`/opsx 归档提案`（或 `/opsx archive`） -3. OpenSpec 会将变更合并到主规格中，并将提案目录移至 `openspec/changes/archive/` - -**示例**：输入 `/opsx 归档提案` - ---- - -# 常见问题与最佳实践 - -**最佳实践**： - -| 问题 | 解决方案 | -|---|---| -| 拷问后忘记在同一会话中发起提案 | 若已关闭对话，从 `CONTEXT.md` 和 `docs/adr/*.md` 中获取历史拷问结果 | -| 提案生成后想修改内容 | 可直接编辑 `openspec/changes//proposal.md`，然后重新运行 `/opsx propose` 更新 | -| 需求文档包含大量图片 | 在拷问时用文字描述图片内容，或上传补充说明文档 | -| 开发中需要新增需求 | 回到步骤 2 重新拷问，再发起新的提案（而非修改当前提案） | -| OpenSpec 命令报错 | 检查 Node.js 版本，重新安装 `@fission-ai/openspec` 修复配置 | - -## 环境与项目结构 - -详见 [references/environment.md](references/environment.md)。 diff --git a/.joyincode/skills/devwiki/references/environment.md b/.joyincode/skills/devwiki/references/environment.md deleted file mode 100644 index 44662fcde..000000000 --- a/.joyincode/skills/devwiki/references/environment.md +++ /dev/null @@ -1,24 +0,0 @@ -# 环境配置参考 - -## 本地环境要求 - -- **Node.js**：版本 >= 20.19.0 -- **OpenSpec CLI**（用于提案管理） - -## OpenSpec 安装 - -```bash -npm install -g @fission-ai/openspec@latest -``` - -安装后可通过技能 `opsx` 初始化 OpenSpec。 - -## 项目目录结构 - -``` -/ -├── docs/ # 存放所有需求文档（原始 Word/PDF 及转换后的 .md） -├── openspec/ # OpenSpec 自动生成的规格文档（勿手动修改） -├── 前端项目代码/ # 前端项目代码（具体框架不限） -└── 后端项目代码/ # 后端项目代码（具体框架不限） -``` diff --git a/.joyincode/skills/jccb/SKILL.md b/.joyincode/skills/jccb/SKILL.md new file mode 100644 index 000000000..fe50b184b --- /dev/null +++ b/.joyincode/skills/jccb/SKILL.md @@ -0,0 +1,154 @@ +--- +name: jccb +description: 项目框架代码库 RAG 与 grep 搜索技能。需要查看后端框架源码时触发 +--- + +# 技能说明 + +本技能通过 MCP 端点连接 jccb MCP 服务，提供框架代码的语义搜索、文件读取和 grep 内容搜索能力，供 AI 编码助手需要查看框架源码时使用（了解框架功能、排查框架问题） + +## 运行方式 + +使用 Python 脚本 `scripts/jccb.py` 调用 MCP 端点，配置集中在脚本同目录 `jccb.config.json`（endpoint / timeout / projectId） + +```bash +python scripts/jccb.py # 列出可用工具 +python scripts/jccb.py method ping # 连通性检查 +``` + +> 工作目录：脚本路径相对 `.joyincode/skills/jccb/` 所在的项目根目录，或使用脚本绝对路径执行。 + +## 工具说明 + +所有工具都需要 `projectId` 参数（取配置文件中的值，也可用命令行 `projectId=xxx` 覆盖，优先），用于定位框架代码仓库。 + +### 1. RAG 语义搜索 + +| 工具 | 参数 | 用途 | +|------|------|------| +| `searchFrameworkCode` | `search` | 从向量库**语义搜索**框架代码，返回代码片段和元数据。File/Class 类型只返回文件路径（使用`grepFileContent`或`readFileContent`工具获取文件内容），其他类型返回指定行范围内容。搜索范围只包含框架的后端代码 | + +### 2. 文件/内容搜索 + +| 工具 | 参数 | 用途 | +|------|------|------| +| `grepFileContent` | `filePath`、`searchContent` | 在框架代码文件中 **grep 搜索**匹配行（content 模式），返回匹配的行号和行内容 | +| `readFileContent` | `filePath`、`startLine`(可选)、`endLine`(可选) | **读取**框架代码文件的指定行范围内容。不传行号则读整个文件 | +| `grepFiles` | `searchContent` | 在框架代码仓库中搜索包含指定内容的文件列表（**files_with_matches** 模式），只返回文件路径，不返回行内容 | +| `grepCodeFile` | `codePath`、`searchContent` | 根据**代码全路径**（如 `com.aa.xx.XXService.java`）在框架代码中 grep 搜索匹配行 | + +## 使用示例 + +### 1. RAG 语义搜索框架代码 + +```bash +python scripts/jccb.py searchFrameworkCode search=用户登录 +``` + +返回 JSON 数组，每条含 language、codeType、summary、codeFile、content。适合用自然语言描述（参考查询词最佳实践章节）搜索框架代码。 + +### 2. grep 搜索指定文件中的匹配行（content 模式） + +```bash +python scripts/jccb.py grepFileContent filePath=src/main/java/com/jupiter/BaseService.java searchContent=public class +``` + +返回 JSON 数组，每条含 line（行号）和 content（行内容）。适合精确定位包含特定内容的行。 + +### 3. 读取文件指定行范围 + +```bash +python scripts/jccb.py readFileContent filePath=src/main/java/com/jupiter/BaseService.java startLine=10 endLine=50 +``` + +返回文件第 10-50 行的文本内容。不传 startLine/endLine 则读整个文件。适合查看完整代码实现。 + +### 4. 搜索仓库中包含内容的文件列表（files_with_matches 模式） + +```bash +python scripts/jccb.py grepFiles searchContent="public class" +``` + +返回 JSON 数组，每条为文件路径（相对于仓库根目录）。适合快速定位哪些文件包含特定内容。排除 .git 目录，最多返回 20 个文件。 + +### 5. 按代码全路径 grep 搜索 + +```bash +python scripts/jccb.py grepCodeFile codePath=com.joyintech.jupiter.common.utils.CommonUtil.java searchContent=public static +``` + +适合按 Java 全限定类名搜索。代码全路径扩展名须与框架后端代码类型一致（JAVA→.java，GO→.go，PY→.py），否则返回不匹配提示。 + +## 使用规则 + +1. **语义搜索优先**：不确定文件路径时先用 `searchFrameworkCode` 用自然语言搜索，从结果中获取文件路径 +2. **精确搜索次之**：已知文件路径用 `grepFileContent` grep 搜索，或用 `readFileContent` 读取文件内容 +3. **搜索文件列表**：不确定文件路径但知道要搜索的内容时用 `grepFiles`（files_with_matches 模式）搜索整个仓库 +4. **按类名搜索**：知道 Java 全限定类名时用 `grepCodeFile` 直接定位文件并 grep 搜索 +5. **参数约定**：`key=value` 按字符串传；纯数字参数（如 startLine）自动转整数；含空格的参数用双引号包裹（PowerShell 语法） +6. 所有工具只搜索项目的**框架代码**（非项目业务代码），通过框架信息定位框架代码仓库 + +## searchFrameworkCode 查询词最佳实践 + +> 基于多轮实测总结的规律，用于构造高质量查询词。所有原则均跨框架通用，不绑定任何特定框架的专有术语。 + +### 查询词构造公式 + +**`目标功能领域词 + 结构语义词 + 期望行为词`**，控制在 3~8 个词。 + +结构语义词（引导向量召回实现细节）：`表结构`、`字段`、`主键`、`删除`、`分页`、`配置`… +期望行为词（命中具体方法体）：`查询`、`保存`、`修改`、`删除`、`批量`… + +构造时的判断标准： +- ✅ 词面宽、能同时命中类名/方法名/注释/注解中的一类或多类（命中面越多样，质量越高） +- ❌ 词面窄且是该框架某个功能独有的叫法时，先确认它是否为框架通用概念，避免查询词只对当前框架有效 + +### 通用原则（跨框架适用） + +1. **避免过于通用的动词**：如 `启动`、`执行`、`处理` 这类词几乎每个框架都有调度器/任务/服务方法使用，命中面过广、噪声高。改用更具体的目标行为组合（如 `启动 提交 引擎` 三词组合代替单个 `启动`）。 +2. **避免混入完整类名/符号名**：符号名会把向量相似度推向单一类，导致重复项暴增、覆盖变窄。需要按类名精确查时直接用 `grepCodeFile`。 +3. **同义词互补**：单次查询往往只命中某一侧实现（如只召回引擎 A 的实现），换同义词再查一次可互补覆盖，多次查询结果合并使用。 +4. **不确定性先探测**：不确定目标功能在代码中的实际叫法时，先用 `grepFiles` 搜索功能相关的通用业务词，从命中的文件/注释中提取框架实际使用的术语，再构造语义查询词——比直接猜更稳。 + +### 已知局限 + +- 向量库只索引**后端代码**，SQL 建表脚本/字段定义**搜不到**；确认表结构请用 `grepFiles` + 建表脚本 +- 返回结果中的 Class/File 类型，仅有文件路径无内容，需要配合 `readFileContent` 读取 + +### 使用场景分级 + +| 目的 | 推荐方式 | +|------|---------| +| 快速定位文件/类入口 | `searchFrameworkCode`，`search=目标功能词+核心行为` | +| 理解某个方法的完整实现 | `searchFrameworkCode`，`search=目标功能词+结构词+方法行为`，命中后 `readFileContent` 读全文件 | +| 确认表/字段/SQL 定义 | **不用语义搜索**，直接 `grepFiles` + 建表脚本 | +| 按已知类名看代码 | `grepCodeFile`，比语义搜索精确 | + +## 常见错误与处理 + +### MCP 服务未启动 + +**现象**：调用任意工具时，请求超时或连接被拒绝 + +**处理方式**：确认 codebase MCP 服务已启动，检查 `jccb.config.json` 中 `endpoint` 配置是否正确 + +### 框架未配置代码库 + +**现象**：返回 `"框架未配置代码库"` + +**处理方式**：请先在JoyinCode管理中台指定项目的具体框架 + +### 框架代码未索引 + +**现象**：`searchFrameworkCode` 返回 `"框架代码未索引到向量库，请先索引框架代码"` + +**处理方式**：需要联系运维人员并提供框架git代码库信息 + +### 文件不存在 + +**现象**：`grepFileContent`/`readFileContent`/`grepCodeFile` 返回 `"文件不存在: {filePath}"` + +**处理方式**：先用 `searchFrameworkCode` 或 `grepFiles` 确认文件路径，再使用正确的文件路径调用 + +## 注意事项 +* 当前搜索的框架代码为截止昨日的最新分支，代码可能会与项目使用的框架版本有差异 diff --git a/.joyincode/skills/jccb/scripts/jccb.config.json b/.joyincode/skills/jccb/scripts/jccb.config.json new file mode 100644 index 000000000..0f2a6b66d --- /dev/null +++ b/.joyincode/skills/jccb/scripts/jccb.config.json @@ -0,0 +1,5 @@ +{ + "endpoint": "https://jc.joyintech.com/jc/mcp/jccb", + "timeout": 30, + "projectId": "0" +} \ No newline at end of file diff --git a/.joyincode/skills/jccb/scripts/jccb.py b/.joyincode/skills/jccb/scripts/jccb.py new file mode 100644 index 000000000..30d66d052 --- /dev/null +++ b/.joyincode/skills/jccb/scripts/jccb.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +codebase MCP 客户端 —— 代码 RAG 搜索与文件/grep 搜索 + +直连 codebase MCP 端点（stateless streamable_http，无需认证）， +运行配置集中在同目录 jccb.config.json。 + +用法： + python jccb.py # 列出可用工具 + python jccb.py searchFrameworkCode search=RAG搜索关键词 # 从向量库搜索框架代码 + python jccb.py grepFileContent filePath=src/Main.java searchContent=public class # 在框架代码文件中 grep 搜索匹配行 + python jccb.py readFileContent filePath=src/Main.java startLine=10 endLine=50 # 读取指定行范围 + python jccb.py grepFiles searchContent=public class # 在框架代码仓库中搜索包含内容的文件列表 + python jccb.py grepCodeFile codePath=com.aa.xx.XXService.java searchContent=public class # 按代码全路径 grep 搜索 + python jccb.py projectId=xxx searchFrameworkCode search=关键词 # 命令行覆盖 projectId + python jccb.py method ping # 通用 MCP 方法调用 + +projectId 来源：命令行 projectId=xxx（优先）> 配置文件（jccb.config.json） + +参数约定：key=value 按字符串传；value 以 { 或 [ 开头时自动按 JSON 解析。 +""" + +import json +import os +import sys +import urllib.request + +CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "jccb.config.json") + +# 强制 stdout/stderr 使用 UTF-8，避免 Windows 控制台 GBK 乱码 +for stream in (sys.stdout, sys.stderr): + try: + stream.reconfigure(encoding="utf-8") + except (AttributeError, ValueError): + pass + + +def load_config() -> dict: + """读取配置文件；缺失或损坏则报错退出。""" + if not os.path.isfile(CONFIG_FILE): + print(f"错误：找不到配置文件 {CONFIG_FILE}", file=sys.stderr) + sys.exit(1) + try: + with open(CONFIG_FILE, "r", encoding="utf-8") as f: + cfg = json.load(f) or {} + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as e: + print(f"错误：配置文件解析失败 {CONFIG_FILE}：{e}", file=sys.stderr) + sys.exit(1) + return cfg + + +CFG = load_config() + + +def _require(key: str) -> str: + """取必填字符串配置项，缺失则报错退出。""" + val = CFG.get(key) + if not isinstance(val, str) or not val.strip(): + print(f"错误：配置缺失必填项 {key}（{CONFIG_FILE}）", file=sys.stderr) + sys.exit(1) + return val.strip() + + +def get_endpoint() -> str: + """端点：仅来自配置文件 endpoint（必填）。""" + return _require("endpoint") + + +def get_timeout() -> int: + """超时：配置 timeout（须为正整数）。""" + t = CFG.get("timeout") + try: + t = int(t) + except (TypeError, ValueError): + print(f"错误：配置 timeout 须为整数（{CONFIG_FILE}）", file=sys.stderr) + sys.exit(1) + if t <= 0: + print(f"错误：配置 timeout 须为正整数（{CONFIG_FILE}）", file=sys.stderr) + sys.exit(1) + return t + + +def get_configured_project_id() -> str: + """读取配置 projectId。""" + val = CFG.get("projectId") + return val.strip() if isinstance(val, str) else "" + + +# 工具名 -> (必填参数列表, 可选参数列表, 描述) +TOOLS = { + # === RAG 语义搜索 === + "searchFrameworkCode": ( + ["search"], + [], + "从向量库中搜索框架代码，返回代码片段和元数据", + ), + # === 文件/内容搜索（GrepSearchMcpTool）=== + "grepFileContent": ( + ["filePath", "searchContent"], + [], + "在框架代码文件中 grep 搜索匹配行（content 模式），返回行号和行内容", + ), + "readFileContent": ( + ["filePath"], + ["startLine", "endLine"], + "读取框架代码文件的指定行范围内容", + ), + "grepFiles": ( + ["searchContent"], + [], + "在框架代码仓库中搜索包含指定内容的文件列表（files_with_matches 模式），只返回文件路径", + ), + "grepCodeFile": ( + ["codePath", "searchContent"], + [], + "根据代码全路径（如 com.aa.xx.XXService.java）在框架代码中 grep 搜索匹配行", + ), +} + +# 常用 MCP 方法说明 +METHODS = { + "initialize": "协议握手（stateless 端点通常可跳过）", + "ping": "连通性检查", + "tools/list": "列出服务器所有工具", + "tools/call": "调用工具（与直接子命令等价）", + "resources/list": "列出服务器资源", +} + + +def resolve_project_id(explicit: str = "") -> str: + """projectId 两个来源：命令行传入（优先）> 配置文件。""" + return explicit or get_configured_project_id() + + +def rpc(payload: dict) -> dict: + """发送 JSON-RPC 请求，返回 result 部分。""" + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request( + get_endpoint(), + data=body, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=get_timeout()) as resp: + data = json.loads(resp.read().decode("utf-8")) + if "error" in data: + raise RuntimeError(f"MCP error: {data['error']}") + return data["result"] + + +def parse_args(items) -> dict: + """key=value -> dict；value 以 { 或 [ 开头时按 JSON 解析。""" + out = {} + for item in items: + if "=" not in item: + continue + key, _, value = item.partition("=") + stripped = value.strip() + if stripped.startswith(("{", "[")): + try: + out[key] = json.loads(stripped) + continue + except json.JSONDecodeError: + pass + # 尝试将纯数字字符串转为整数（用于 startLine/endLine） + if stripped.lstrip("-").isdigit(): + out[key] = int(stripped) + else: + out[key] = value + return out + + +def list_all(project_id: str = "") -> int: + print(f"codebase MCP 端点：{get_endpoint()}") + print(f"配置文件：{CONFIG_FILE}") + if project_id: + print(f"projectId：{project_id}（来自配置文件或命令行）") + else: + print("projectId：未解析到（请在配置文件中写入，或用 projectId=xxx 传入）") + + print("\n== RAG 语义搜索 ==") + for name, (required, optional, desc) in TOOLS.items(): + if name.startswith("search"): + params = ", ".join(required) + " (必填)" + if optional: + params += " | " + ", ".join(optional) + " (可选)" + print(f"- {name}: {desc}\n 参数: {params}") + + print("\n== 文件/内容搜索 ==") + for name, (required, optional, desc) in TOOLS.items(): + if not name.startswith("search"): + params = ", ".join(required) + " (必填)" + if optional: + params += " | " + ", ".join(optional) + " (可选)" + print(f"- {name}: {desc}\n 参数: {params}") + + print("\n== 通用 MCP 方法（method 子命令）==") + for name, desc in METHODS.items(): + print(f"- {name}: {desc}") + + print("\n示例：") + print(" python jccb.py searchFrameworkCode search=用户登录") + print(" python jccb.py grepFileContent filePath=src/Main.java searchContent=public") + print(" python jccb.py readFileContent filePath=src/Main.java startLine=10 endLine=50") + print(" python jccb.py grepFiles searchContent=public class") + print(" python jccb.py grepCodeFile codePath=com.aa.xx.XXService.java searchContent=public class") + return 0 + + +def call_tool(name: str, args: dict) -> int: + result = rpc({"jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": name, "arguments": args}}) + if result.get("isError"): + print(f"错误：{result}", file=sys.stderr) + return 1 + for block in result.get("content", []): + if block.get("type") == "text": + print(block.get("text", "")) + else: + print(json.dumps(block, ensure_ascii=False, indent=2)) + return 0 + + +def call_method(method: str, args: dict) -> int: + result = rpc({"jsonrpc": "2.0", "id": 1, "method": method, "params": args}) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +def main() -> int: + argv = sys.argv[1:] + if not argv: + return list_all(resolve_project_id()) + + first, rest = argv[0], argv[1:] + + # method 子命令 + if first == "method": + if not rest: + print("用法: python jccb.py method <方法名> [key=value...]", file=sys.stderr) + return 1 + return call_method(rest[0], parse_args(rest[1:])) + + # 工具调用 + args = parse_args(argv[1:]) + explicit = args.get("projectId", "") + pid = resolve_project_id(explicit) + if not pid: + print("错误：未解析到 projectId。", file=sys.stderr) + print("请在配置文件 jccb.config.json 中写入 projectId，", file=sys.stderr) + print("或用 projectId=xxx 在命令中传入。", file=sys.stderr) + return 1 + args["projectId"] = pid + + if first not in TOOLS: + print(f"未知工具: {first}", file=sys.stderr) + print(f"可用工具: {', '.join(TOOLS)}", file=sys.stderr) + print("通用方法请用: python jccb.py method <方法名>", file=sys.stderr) + return 1 + + # 校验必填参数 + required, optional, desc = TOOLS[first] + missing = [p for p in required if p not in args] + if missing: + print(f"错误：工具 {first} 缺少必填参数: {', '.join(missing)}", file=sys.stderr) + print(f" 必填: {', '.join(required)}", file=sys.stderr) + if optional: + print(f" 可选: {', '.join(optional)}", file=sys.stderr) + return 1 + + return call_tool(first, args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.joyincode/skills/markitdown/SKILL.md b/.joyincode/skills/markitdown/SKILL.md index 6787e2cba..995783448 100644 --- a/.joyincode/skills/markitdown/SKILL.md +++ b/.joyincode/skills/markitdown/SKILL.md @@ -1,10 +1,14 @@ --- name: markitdown -description: 将Word文档转为markdown文件 +description: 将 Word/PDF/PPT/Excel 等文档转为 Markdown 文件。用户提到 markitdown、word转markdown、word转md、convert_to_markdown 时触发 --- -# Word转Markdown -## 步骤1：上传文件 +# 文档转 Markdown + +三步流程：**curl 上传文件拿 URI → 脚本调用 MCP 转换 → stdout 重定向保存 .md**。脚本仅标准库实现、内部强制 UTF-8 输出（Windows 控制台无乱码）；端点/超时配置在 `scripts/markitdown.config.json`。 + +## 步骤1：上传文件（获取 URI） + 将文件通过接口`https://jc.joyintech.com/jupiter-ai/codehelper/markitdown/upload`上传 响应示例： @@ -21,21 +25,27 @@ curl -s -X POST -F "file=@<源文件路径>" "https://jc.joyintech.com/jupiter-a > Windows PowerShell 下用 `curl.exe`（`curl` 是 `Invoke-WebRequest` 别名） ## 步骤2：调用 MCP 转换服务 -- 使用MCP服务`https://jc.joyintech.com/jc/mcp/markitdown`的 convert_to_markdown 工具 -- 将步骤1返回的 `data` 值作为 `uri` 参数 -- 请求头为 `Accept: application/json, text/event-stream` -### ⚠️ **乱码问题** -**不要用 PowerShell 的 `Invoke-RestMethod` / `Invoke-WebRequest` 调用此服务。** +**推荐：用 `-o` 直接输出到文件**（脚本以 UTF-8 写入，不受 Shell 重定向编码影响）： + +```bash +python .joyincode/skills/markitdown/scripts/markitdown.py convert_to_markdown uri=file:///data/2026xxxx/xxx.docx -o document.md +``` -根因：PowerShell 5.1 默认不按 UTF-8 解码 HTTP 响应体，`Get-Content`（即使指定 `-Encoding UTF8`）输出到控制台时也按 GBK 编码，会把 MCP 返回的 UTF-8 中文显示成乱码，误导你以为"服务端返回乱码"，实际是客户端显示层问题。 +> 以上命令从项目根目录执行。脚本路径为 `.joyincode/skills/markitdown/scripts/markitdown.py`（项目根目录下没有 `scripts/` 目录），也可改用脚本绝对路径。 -## 步骤3：提取并保存 Markdown -- 响应是 JSON-RPC 格式，Markdown 内容在 `result.content[0].text` 中 -- 必须以 UTF-8 读取步骤2的临时响应文件再解析，避免编码二次污染 -- 保存后删除临时文件 +如需 stdout 输出（管道/重定向场景），也可不加 `-o`： + +```bash +python .joyincode/skills/markitdown/scripts/markitdown.py convert_to_markdown uri=file:///data/2026xxxx/xxx.docx > document.md +``` + +> 注意：Windows PowerShell 下 `>` 重定向会把输出转成 UTF-16 并按 GBK 解码，导致中文乱码。 + +无需手动解析 JSON-RPC 或处理临时响应文件编码——脚本已封装。 + +## 步骤3（可选）：下载图片 -### 图片下载 若生成的md文件中包含图片 `![xxxx](docx_images/xxxx.png)`，可调用以下接口批量获取图片文件 接口地址： @@ -48,7 +58,15 @@ curl -s -X POST -F "file=@<源文件路径>" "https://jc.joyintech.com/jupiter-a - 成功返回 `application/zip` 文件流。部分文件不存在或路径非法时自动跳过，仅打包有效文件 - 若无文件可下载，返回json：`{ "code":"没有可下载的文件", "data":"4d7c11690c0c468e8ce8246fb7c268dc", "message":"没有可下载的文件" }` -### 排障指引 -若看到乱码，**优先怀疑客户端读取/显示层编码，而非服务端**： -- Windows 下用 Read 工具读取步骤2保存的响应文件——若内容正确，说明服务端无问题 -- **不要因此去安装本地 markitdown 包绕路转换。** +## 使用规则 + +1. **先上传再转换**：脚本不含上传，必须先经步骤1拿到 `file:///...` URI +2. **乱码已由脚本解决**：直接重定向到文件即可；不要用 PowerShell `Invoke-RestMethod`/`Invoke-WebRequest` 手动调 MCP（GBK 乱码），也不要安装本地 markitdown 包绕路转换 +3. **通用方法**：`python .joyincode/skills/markitdown/scripts/markitdown.py method <方法名>` 可调用任意 MCP 方法（如 `ping`/`tools/list`）；无参数运行列出全部工具与方法 + +## 故障排查 + +| 现象 | 处理 | +|---|---| +| 连接失败/超时/406 | `python .joyincode/skills/markitdown/scripts/markitdown.py method ping` 验证连通性；确认 `markitdown.config.json` 的 endpoint 为 `https://jc.joyintech.com/jc/mcp/markitdown`、timeout 足够 | +| 报错找不到配置文件 | 从项目根目录运行，或用脚本绝对路径 | diff --git a/.joyincode/skills/markitdown/scripts/markitdown.config.json b/.joyincode/skills/markitdown/scripts/markitdown.config.json new file mode 100644 index 000000000..a9f871fd3 --- /dev/null +++ b/.joyincode/skills/markitdown/scripts/markitdown.config.json @@ -0,0 +1,4 @@ +{ + "endpoint": "https://jc.joyintech.com/jc/mcp/markitdown", + "timeout": 120 +} diff --git a/.joyincode/skills/markitdown/scripts/markitdown.py b/.joyincode/skills/markitdown/scripts/markitdown.py new file mode 100644 index 000000000..26f718c1a --- /dev/null +++ b/.joyincode/skills/markitdown/scripts/markitdown.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +markitdown MCP 客户端 —— 文档转 Markdown + +直连 markitdown MCP 端点（stateless streamable_http，无需认证）， +运行配置集中在同目录 markitdown.config.json。 + +用法： + python markitdown.py # 列出可用工具与方法 + python markitdown.py convert_to_markdown uri=file:///data/2026xxxx/xxx.docx # 转换文档URI为Markdown + python markitdown.py convert_to_markdown uri=file:///data/2026xxxx/xxx.docx -o out.md # 转换并直接写入文件（UTF-8，避免Shell重定向乱码） + python markitdown.py method ping # 通用 MCP 方法调用 + python markitdown.py method tools/list # 列出服务器所有工具 + +转换所需的文件 URI 来源：先调用上传接口 + POST https://jc.joyintech.com/jupiter-ai/codehelper/markitdown/upload + 取响应 data 字段（file:///... URI）作为本脚本的 uri 参数。 + +参数约定：key=value 按字符串传；value 以 { 或 [ 开头时自动按 JSON 解析。 +""" + +import json +import os +import sys +import urllib.request + +CONFIG_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "markitdown.config.json") + +# 强制 stdout/stderr 使用 UTF-8，避免 Windows 控制台 GBK 乱码 +# （markitdown 转换结果含大量中文，客户端显示层必须按 UTF-8 解码） +for stream in (sys.stdout, sys.stderr): + try: + stream.reconfigure(encoding="utf-8") + except (AttributeError, ValueError): + pass + + +def load_config() -> dict: + """读取配置文件；缺失或损坏则报错退出。""" + if not os.path.isfile(CONFIG_FILE): + print(f"错误：找不到配置文件 {CONFIG_FILE}", file=sys.stderr) + sys.exit(1) + try: + with open(CONFIG_FILE, "r", encoding="utf-8") as f: + cfg = json.load(f) or {} + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as e: + print(f"错误：配置文件解析失败 {CONFIG_FILE}：{e}", file=sys.stderr) + sys.exit(1) + return cfg + + +CFG = load_config() + + +def _require(key: str) -> str: + """取必填字符串配置项，缺失则报错退出。""" + val = CFG.get(key) + if not isinstance(val, str) or not val.strip(): + print(f"错误：配置缺失必填项 {key}（{CONFIG_FILE}）", file=sys.stderr) + sys.exit(1) + return val.strip() + + +def get_endpoint() -> str: + """端点：仅来自配置文件 endpoint（必填）。""" + return _require("endpoint") + + +def get_timeout() -> int: + """超时：配置 timeout（须为正整数）。""" + t = CFG.get("timeout") + try: + t = int(t) + except (TypeError, ValueError): + print(f"错误：配置 timeout 须为整数（{CONFIG_FILE}）", file=sys.stderr) + sys.exit(1) + if t <= 0: + print(f"错误：配置 timeout 须为正整数（{CONFIG_FILE}）", file=sys.stderr) + sys.exit(1) + return t + + +# 工具名 -> (参数列表, 描述) +TOOLS = { + "convert_to_markdown": (["uri"], "将文档(Word/PDF/PPT/Excel等)URI转换为Markdown"), +} + +# 常用 MCP 方法说明 +METHODS = { + "initialize": "协议握手（stateless 端点通常可跳过）", + "ping": "连通性检查", + "tools/list": "列出服务器所有工具", + "tools/call": "调用工具（与直接子命令等价）", + "resources/list": "列出服务器资源", +} + + +def rpc(payload: dict) -> dict: + """发送 JSON-RPC 请求，返回 result 部分。""" + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + req = urllib.request.Request( + get_endpoint(), + data=body, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=get_timeout()) as resp: + data = json.loads(resp.read().decode("utf-8")) + if "error" in data: + raise RuntimeError(f"MCP error: {data['error']}") + return data["result"] + + +def parse_args(items) -> dict: + """key=value -> dict；value 以 { 或 [ 开头时按 JSON 解析。""" + out = {} + for item in items: + if "=" not in item: + continue + key, _, value = item.partition("=") + stripped = value.strip() + if stripped.startswith(("{", "[")): + try: + out[key] = json.loads(stripped) + continue + except json.JSONDecodeError: + pass + out[key] = value + return out + + +def extract_output_flag(argv: list) -> tuple: + """提取 -o/--output 输出文件参数，返回 (剩余参数, 输出路径或None)。""" + out = None + rest = [] + i = 0 + while i < len(argv): + arg = argv[i] + if arg in ("-o", "--output") and i + 1 < len(argv): + out = argv[i + 1] + i += 2 + continue + rest.append(arg) + i += 1 + return rest, out + + +def list_all() -> int: + print(f"markitdown MCP 端点：{get_endpoint()}") + print(f"配置文件：{CONFIG_FILE}") + + print("\n== 工具（tools/call）==") + for name, (params, desc) in TOOLS.items(): + sig = ", ".join(params) + " (必填)" + print(f"- {name}: {desc}\n 参数: {sig}") + + print("\n== 通用 MCP 方法（method 子命令）==") + for name, desc in METHODS.items(): + print(f"- {name}: {desc}") + + print("\n说明：") + print(" 转换所需的文件 URI 需先经上传接口获取：") + print(" POST https://jc.joyintech.com/jupiter-ai/codehelper/markitdown/upload") + print(" 取响应 data 字段（file:///... URI）作为 uri 参数传入。") + + print("\n示例：") + print(" python markitdown.py convert_to_markdown uri=file:///data/2026xxxx/xxx.docx") + print(" python markitdown.py convert_to_markdown uri=file:///data/2026xxxx/xxx.docx -o out.md") + print(" python markitdown.py method ping") + print(" python markitdown.py method tools/list") + return 0 + + +def call_tool(name: str, args: dict, output_file: str = None) -> int: + result = rpc({"jsonrpc": "2.0", "id": 1, "method": "tools/call", + "params": {"name": name, "arguments": args}}) + if result.get("isError"): + print(f"错误：{result}", file=sys.stderr) + return 1 + texts = [] + for block in result.get("content", []): + if block.get("type") == "text": + texts.append(block.get("text", "")) + else: + texts.append(json.dumps(block, ensure_ascii=False, indent=2)) + content = "\n".join(texts) + if output_file: + try: + with open(output_file, "w", encoding="utf-8", newline="") as f: + f.write(content) + except OSError as e: + print(f"错误：写入输出文件失败 {output_file}：{e}", file=sys.stderr) + return 1 + print(f"已写入 {output_file}（{len(content.encode('utf-8'))} 字节）") + else: + print(content) + return 0 + + +def call_method(method: str, args: dict) -> int: + result = rpc({"jsonrpc": "2.0", "id": 1, "method": method, "params": args}) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +def main() -> int: + argv = sys.argv[1:] + if not argv: + return list_all() + + argv, output_file = extract_output_flag(argv) + + first, rest = argv[0], argv[1:] + + # method 子命令 + if first == "method": + if not rest: + print("用法: python markitdown.py method <方法名> [key=value...]", file=sys.stderr) + return 1 + return call_method(rest[0], parse_args(rest[1:])) + + # 工具调用 + args = parse_args(argv[1:]) + + if first not in TOOLS: + print(f"未知工具: {first}", file=sys.stderr) + print(f"可用工具: {', '.join(TOOLS)}", file=sys.stderr) + print("通用方法请用: python markitdown.py method <方法名>", file=sys.stderr) + return 1 + + return call_tool(first, args, output_file) + + +if __name__ == "__main__": + sys.exit(main()) diff --git "a/.joyincode/skills/\346\234\252\347\273\217\346\216\210\346\235\203\347\246\201\346\255\242\345\212\250\345\267\262\346\234\211\344\273\243\347\240\201/SKILL.md" "b/.joyincode/skills/\346\234\252\347\273\217\346\216\210\346\235\203\347\246\201\346\255\242\345\212\250\345\267\262\346\234\211\344\273\243\347\240\201/SKILL.md" new file mode 100644 index 000000000..fe1737ed1 --- /dev/null +++ "b/.joyincode/skills/\346\234\252\347\273\217\346\216\210\346\235\203\347\246\201\346\255\242\345\212\250\345\267\262\346\234\211\344\273\243\347\240\201/SKILL.md" @@ -0,0 +1,27 @@ +# 代码修改权限控制 Skill + +## 核心原则 +在未经用户明确、清晰授权的情况下，我（AI助手）**不得主动修改、删除或重写用户已有的任何代码**。 + +## 详细行为准则 + +### 1. 绝对不能做的事（红线） +- **未授权修改**：用户未明确要求时，**绝对不修改**任何已有的代码行或文件结构。 +- **直接替换**：**绝对不**在用户未同意的情况下，用一段全新的代码直接覆盖用户现有的代码。 +- **擅自重构**：**绝对不**对用户代码进行大规模重构，除非用户明确提出重构需求。 + +### 2. 允许的例外情况 +以下情况可以不经用户确认直接操作，但仅限于演示或临时环境： +- **生成新文件**：创建用户明确要求的新文件（如 `UserController.java`）。 +- **添加新代码**：在用户指定的需求中，添加用户要求的新方法或代码块，且不影响现有业务逻辑。 + +### 3. 互动流程示例 +- **错误示范**： + - 用户：“这个`Service`类代码有点长” + - AI：*立即开始重写整个Service类并输出新代码* ❌ + +- **正确示范**： + - 用户：“这个`Service`类代码有点长” + - AI：“我注意到您的`UserService`类有300行，建议可以按职责拆分为`UserQueryService`和`UserUpdateService`。这样做的好处是... 如果您同意这个方案，我可以帮您重构。” ✅ + - 用户：“好的，请帮我拆分。” + - AI：*执行重构操作* ✅ \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 2c2d5dd36..0e626510a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,7 +13,7 @@ trading. This repo is a performance-oriented fork of the original metaprogramming** in favor of explicit mixin + factory initialization while keeping the public API compatible. -- **Version**: `1.3.0` (see `backtrader/version.py`) +- **Version**: `1.4.0` (see `backtrader/version.py`) - **License**: GPLv3 - **Python**: 3.8–3.13 (classifiers in `setup.py`; 3.11 recommended) - **Not on PyPI** — install from source only. @@ -74,12 +74,13 @@ tiers by **measured per-file duration**, applied dynamically at collection time (no test files are edited): ```bash -make test-fast # ~3.5 min: all non-strategy tests + fastest ~35% of - # strategy tests. Daily "did I break anything" loop. - # == pytest tests -m "not slow" -n 8 -q +make test-fast # parallel non-performance tests + serial wall-clock + # microbenchmarks; excludes slowest ~65% of strategy tests. + # Daily "did I break anything" loop. make test-slow # the slowest ~65% strategy tests test-fast skips make test-strategies # all 1,271 strategy regression tests (~9 min) -make test-all # entire suite in parallel (~10 min) +make test-all # parallel functional suite + serial wall-clock microbenchmarks +make test-performance # wall-clock microbenchmarks without xdist make test-coverage # coverage report # Single test, verbose: @@ -99,6 +100,15 @@ How the split works: - Refresh timings after adding/removing strategy tests: `python scripts/refresh_strategy_durations.py`. +Wall-clock microbenchmarks and time-bounded latency contracts have a separate +serial lane: those tests are explicitly skipped under xdist or coverage tracing +and `make test-performance` runs them without either. Its short RSS stress +profile uses a separate fresh pytest process so suite-import RSS cannot be +mistaken for the profile's process-tree budget. `make test-fast` and +`make test-all` include that serial lane after their parallel functional tests, +preserving each performance contract without treating worker scheduling noise +as an application regression. + ### Choosing which `backtrader` to test against Running pytest from the repo root resolves `import backtrader` to the **local @@ -191,9 +201,15 @@ Access patterns: `data.close[0]` (current bar), `data.close[-1]` (previous). - `feed.py` + `feeds/` (17 files) — CSV, pandas, IB, CCXT, etc.; `resamplerfilter.py` for resample/replay. - `broker.py` + `brokers/` — order matching and portfolio state. -- `cerebro.py` (~2,440 lines) — orchestrator. `run()` → `runstrategies()` → - `_runonce()` (vectorized) or `_runnext()` (event-driven). Tick-level mode is - also supported. +- `cerebro.py` (~830 lines, public facade) + `_cerebro/` private mixin package + (9 files, iteration 28 split) — orchestrator. The facade keeps the `Cerebro` + class definition (params/descriptors/`__init__`/`run`/pickle protocol) and + `OptReturn`; `registry/notifications/lifecycle/channel/execution` hold + configuration, dispatch and orchestration; `runnext`/`runonce` hold the + four engine loops (hot paths — verbatim-moved, see + `docs/_internal/opts/requirements/迭代28-Cerebro模块化拆分/`). + `run()` → `runstrategies()` → `_runonce()` (vectorized) or `_runnext()` + (event-driven). Tick-level mode is also supported. ### Indicator registration & multi-data clocks (high-bug-risk area) @@ -237,7 +253,8 @@ Data Feed(s) → Cerebro → Strategy → Indicators / Observers / Analyzers ``` backtrader/ core library - cerebro.py strategy.py indicator.py analyzer.py observer.py broker.py feed.py + cerebro.py (facade) + _cerebro/ (private engine mixins) strategy.py + indicator.py analyzer.py observer.py broker.py feed.py metabase.py parameters.py lineroot.py linebuffer.py lineseries.py lineiterator.py dataseries.py indicators/ analyzers/ observers/ feeds/ brokers/ filters/ sizers/ signals/ @@ -253,6 +270,11 @@ docs/ Sphinx docs (EN + ZH) + design/bug notes scripts/ optimize_code.sh, refresh_strategy_durations.py, run_strategy_branch_compare.py, … studies/ research/diagnostic scripts (e.g. branch_compare/) +examples/012_1_midfreq_cross_exchange/ mid-frequency OKX/Binance perpetual example +examples/012_2_event_driven_cross_exchange/ event-driven OKX/Binance perpetual candidate +examples/013_3_sa_midfreq_simnow/ controlled CTP/SimNow SA mid-frequency example +examples/strategy-candidate-manifest.json hash-bound research/demo admission manifest +examples/strategy_candidate_approval.py candidate-specific receipt/provenance policy Makefile pyproject.toml setup.py pytest.ini requirements.txt conftest.py ``` @@ -260,6 +282,66 @@ The three AI products are not vendored and are not Git submodules. Make product changes, packaging releases, and product-specific acceptance changes in their respective repositories; this repository only links to them from its README. +The cross-exchange arbitrage examples use `BtApiStore.getdata()` / `BtApiFeed` +with `orderbook_as_ticks=True` and `TimeFrame.Ticks`. Native `notify_orderbook` +callbacks drive `bt.Strategy.buy/sell` and `notify_order`; `BtApiBroker` routes +demo orders through public `BtApi` methods with `normalized=True`. +The SDK owns venue schemas, request mapping and optional execution-session state +(durable intents, unique client IDs, uncertain-order reconciliation and fees). +`bt_api_py.cross_venue` owns only provider-neutral, stateless typed execution +planning: quantity lattices, executable VWAP, cost accounting, funding schedule +validation and normalized orderbook evidence. It consumes SDK contracts and +does not own a client, account, order, pair state, alpha, or compensation policy. +The store holds `BtApi` directly and only maps framework orders, references and +native market-data objects; there is no second Backtrader trading client. +`examples/strategy_candidate_approval.py` binds the two example candidates' +manifest, offline receipt and source provenance. It is example admission policy, +not a Backtrader utility or SDK protocol. +OKX endpoint selection belongs to the SDK through +`api_region=global|eea|us|tr`: REST plus public/private/business WebSockets use +one atomic region/environment profile. Global/EEA/US support production and +demo; TR currently supports only production, so `tr+demo` fails before network +I/O. OKX 50119 proves that the selected credential/domain combination was +rejected; by itself it does not distinguish region, key, secret, passphrase, +expiry, or permission causes. +Funding is a typed SDK read model. `BtApiStore` refreshes it on a separate +single-concurrency read-only lane with request coalescing, TTL/schedule-boundary +expiry, and generation fencing; strategy callbacks only read the local cache. +This snapshot supports entry reserves and settlement-window risk. The SDK does +not yet expose a unified, pagination-complete, account-bound OKX/Binance funding +cashflow ledger, so a cycle crossing settlement cannot claim complete realized +net PnL. The production status is +`PRODUCTION_BLOCKED_ACTUAL_FUNDING_CASHFLOW_LEDGER`: venue-level single-page raw +income/bills parsers do not prove pagination coverage, identity, deduplication, +aggregation, settlement latency, or a complete empty result. Idle risk also +advances without a new bar through `notify_idle` polling. +The `exchange_kwargs` and `symbol_routes` +configuration supports multiple providers in a single broker. Amounts remain native units +(OKX contracts / Binance BTC); strategy sizing uses metadata multipliers. +The examples require independently verified dual-side/hedge mode and maintain +long/short legs separately; no net-position fallback is accepted. `shadow` uses +public production books with zero orders/fills/PnL, `paper-live` uses public books +with local hypothetical fills, and only `demo` can submit exchange orders. Demo +writes additionally require a strategy-specific, hash-bound approval receipt. +Both Iteration 21 frozen candidates failed their pre-OOS calibration cost screen, +so their `paper-live` simulated-fill and `demo` order paths remain prohibited; +read-only shadow and demo preflight remain available. Any new economic attempt +requires a new candidate ID, preregistration, and untouched holdout. +Credentials are kept in each example's ignored `.env`. Deterministic `replay` +reports are formula fixtures with zero orders/fills and no PnL; the native +Store/Feed/Cerebro/Broker path is tested separately. The second candidate is +classified as event-driven and remains `HFT FAIL/NOT_ADMITTED` until end-to-end +latency, queue and real-fill evidence exists. + +The Iteration 22 SA example uses one authoritative `BtApiFeed` to dispatch CTP +quote events and form watermark-closed one-minute bars. CTP trading admission +requires typed terminal account/position/order/trade/reference queries bound to +one stable connection generation and account fingerprint. `replay` is an +offline zero-write path, `shadow` is read-only, and `simnow` additionally +requires a hash-bound approval receipt plus a complete first-set observation +gate. The example never treats replay output or a single SimNow day as evidence +that the strategy is profitable. + ## Tests - `tests/functional/strategies/` holds 1,271 inlined regression tests across ~30 @@ -303,6 +385,34 @@ respective repositories; this repository only links to them from its README. `studies/branch_compare/` + `scripts/run_strategy_branch_compare.py` with `TradeLogger` is the established way to localize divergences). +## Logging (iteration 29) + +- Single entry point `backtrader/utils/log_message.py` (`get_logger`, + `configure_logging`, throttled storm suppression). See + `docs/LOGGING_GUIDELINES.md`; baseline catalogs are regenerable via + `python scripts/scan_logging_baseline.py --out `. +- Default silence: nothing is emitted or written until + `configure_logging(...)` is called (protected by tests). +- Split-file layout (opt-in): `configure_logging(level="INFO", + log_dir="logs")` writes `logs/