diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6dde7be49..68a8571e4 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -28,7 +28,7 @@ git checkout -b ```bash # 自动打通虚拟环境并注满燃油（安装源码及开发依赖） uv venv -uv pip install -r requirements-dev.txt -e ./ +uv pip install -r .github/requirements-dev.txt -e ./ ``` ### 2. 空中改造 (Coding) @@ -62,7 +62,9 @@ uv run python -m unittest - 📘 **`docs` 航道 (文档维护)**：本航道直通项目专属的 **[在线文档 (Read the Docs)](https://jmcomic.readthedocs.io/zh-cn/latest/)**。所有不涉及功能性代码修改的**纯粹文档更新**，请直接提交至此分支。入线后它将即刻触发并全网同步部署，绕过复杂的流水安检。需要注意的是，`docs` 航道的进度通常会超前于主系统 (`master`)，塔台会在新版发布或定期检修时统一将二者对齐同步。 -- 🚫 **禁飞区 (禁止直飞 master)**：为防止外部航班意外触发 `release_auto.yml` 这个威力巨大的自动发版工作流，**本项目不接受任何直接指向 `master` 分支的 PR**。所有的新功能与代码改造必须经由 `dev` 航道降落并完成试飞，新版本号的最终敲定与发布由塔台统一操控。 +- 🚫 **禁飞区 (普通 PR 禁止直飞 master)**：为防止外部航班意外触发 `release_auto.yml` 这个威力巨大的自动发版工作流，所有的功能、重构、修复、文档和普通维护 PR 都不得直接指向 `master`。这些改动必须先经由 `dev` 或 `docs` 航道降落，完成对应试飞后再由塔台统一调度。 + +- 🚀 **发版专线 (仅限版本发布)**：`master` 只接收由塔台（维护者）从 `dev` 发起的正式发版 PR。该 PR 必须同时完成版本号更新，并在 `CHANGELOG.md` 中准备好对应版本的带日期发布记录；任意一项缺失，都不得指向或合并到 `master`。 PR 申请后，我会尽快 review 各位机长的代码并反馈通讯。再次感谢你的付出！🎉 @@ -70,22 +72,39 @@ PR 申请后，我会尽快 review 各位机长的代码并反馈通讯。再次 ## 🚀 发版巡航 (Release Process) -项目使用自动化流水线进行发版，此流程仅由塔台（维护者）在合并至 `master` 分支时触发。 +项目使用自动化流水线进行发版，此流程仅由塔台（维护者）通过 `dev → master` 发版专线触发。 ### 1. 触发条件 当代码被推送（或 PR 合并）至 `master` 分支，且 **Commit Message 以 `v` 开头**时，GitHub Actions 会自动启动 `release_auto.yml` 工作流，执行以下操作： -1. 自动根据 Commit Message 创建 GitHub Release 标签。 -2. 自动构建项目并发布至 [PyPI](https://pypi.org/project/jmcomic/)。 +1. 自动根据 Commit Message 中的版本号创建 GitHub Release 标签。 +2. 从 `CHANGELOG.md` 对应版本段生成 GitHub Release 正文。 +3. 在 Actions 日志中输出目标版本、Changelog 条目数量和正文来源，供塔台在正式发布前复核。 +4. 自动构建项目并发布至 [PyPI](https://pypi.org/project/jmcomic/)。 ### 2. Commit 格式指令 (仅针对维护者) -当塔台（维护者）准备好发布新版本并合入 `master` 时，需要根据 `.github/release.py` 的解析要求使用特定格式的 Commit Message： +当塔台（维护者）准备好发布新版本并合入 `master` 时，必须先完成以下发版检查： + +1. 更新 `src/jmcomic/__init__.py` 中的 `__version__`。 +2. 在 `CHANGELOG.md` 中添加同版本的 `## [版本号] - YYYY-MM-DD` 段落，并确认自上一个版本以来的重要改动均已记录。 +3. 确认版本号与 Changelog 均已包含在发版 PR 中，再使用以下 Commit Message 合并到 `master`。 -**格式：** `v<版本号>: <更新项1>; <更新项2>; ...` +**格式：** `v<版本号>: <简短发布说明>` -* **示例：** `v2.1.0: 修复搜索解析异常; 优化多线程下载效率; 新增插件系统` +* **示例：** `v2.7.4: 发布下载清单功能` * **黑匣子解析逻辑：** * **Tag**：冒号 `:` 前的内容（如 `v2.1.0`）。 - * **Body**：冒号 `:` 后的内容，程序会将分号 `;` 分割的每一项转化为有序列表。 + * **版本校验**：Commit Message 中的版本必须与 `jmcomic.__version__` 一致。 + * **Body**：GitHub Release 正文完全来自 `CHANGELOG.md` 的同版本段落；冒号后的文字只用于说明本次发版提交，不再充当 Release 正文。 + +### 3. 备降与人工复飞 + +如果符合规范的发版提交已经进入 `master`，但自动流水线因为 Changelog 遗漏、格式错误或版本不匹配而在创建 Tag、GitHub Release 和 PyPI 包之前中止，塔台无需再制造第二条 `v版本号:` 发版提交： + +1. 在 `master` 补充一条普通修正提交，只修正当前版本的版本号或 Changelog 发版资料，不夹带新的功能代码。 +2. 在 GitHub Actions 中选择 `master` 分支，手动运行 `Manual Release & Publish`。 +3. 手动流水线会直接读取 `src/jmcomic/__init__.py` 中的版本号，并使用 `CHANGELOG.md` 的同版本段落完成发布。 + +人工复飞仅用于修复自动发版的前置校验失败，不替代正常的 `dev → master` 发版专线，也不得用于绕过发版 PR 的版本与 Changelog 检查。 --- diff --git a/.github/release.py b/.github/release.py index 8cc8eaf04..6b3d5b162 100644 --- a/.github/release.py +++ b/.github/release.py @@ -1,45 +1,104 @@ +"""Build GitHub Release metadata from a release commit and changelog.""" + +import ast import os -import sys import re +import sys +from pathlib import Path +from typing import Optional, Tuple -def add_output(k, v): - cmd = f'echo "{k}={v}" >> $GITHUB_OUTPUT' - print(cmd, os.system(cmd)) - +ROOT_DIR = Path(__file__).resolve().parent.parent +VERSION_FILE = Path("src/jmcomic/__init__.py") +CHANGELOG_FILE = Path("CHANGELOG.md") +RELEASE_BODY_FILE = Path("release_body.txt") +RELEASE_SUBJECT_PATTERN = re.compile(r"^v(?P\d+\.\d+\.\d+):(?:\s.*)?$") +VERSION_HEADING_PATTERN = re.compile( + r"^## \[(?P[^]]+)] - (?P\d{4}-\d{2}-\d{2})\s*$", + re.MULTILINE, +) -def parse_body(body): - if ';' not in body: - return body - parts = body.split(";") - points = [] - for i, e in enumerate(parts): - e: str = e.strip() - if e == '': +def read_source_version(path: Path) -> str: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in tree.body: + if not isinstance(node, ast.Assign) or len(node.targets) != 1: continue - points.append(f'{i + 1}. {e}') + target = node.targets[0] + if isinstance(target, ast.Name) and target.id == "__version__": + if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str): + return node.value.value + raise ValueError(f"Static __version__ assignment not found in {path}") + + +def read_release_version(commit_message: str) -> str: + subject = commit_message.splitlines()[0].strip() if commit_message else "" + match = RELEASE_SUBJECT_PATTERN.fullmatch(subject) + if match is None: + raise ValueError(f"Release commit must match v{{version}}: summary, got: {subject}") + return match.group("version") + + +def extract_release_body(changelog: str, version: str) -> str: + matches = [match for match in VERSION_HEADING_PATTERN.finditer(changelog) if match.group("version") == version] + if not matches: + raise ValueError(f"Changelog section not found: ## [{version}] - YYYY-MM-DD") + if len(matches) > 1: + raise ValueError(f"Duplicate changelog sections found for version {version}") + + match = matches[0] + next_heading = re.search(r"^## \[", changelog[match.end():], re.MULTILINE) + section_end = match.end() + next_heading.start() if next_heading else len(changelog) + body = changelog[match.end():section_end].strip() + if not body: + raise ValueError(f"Changelog section for version {version} is empty") + return body + + +def count_release_entries(body: str) -> int: + return sum(1 for line in body.splitlines() if line.lstrip().startswith("- ")) + - return '\n'.join(points) +def build_release_metadata( + commit_message: Optional[str] = None, + root_dir: Optional[Path] = None, +) -> Tuple[str, str]: + root_dir = root_dir or ROOT_DIR + source_version = read_source_version(root_dir / VERSION_FILE) + if commit_message is not None: + release_version = read_release_version(commit_message) + if release_version != source_version: + raise ValueError( + f"Version mismatch: release commit={release_version}, __init__.py={source_version}" + ) + changelog = (root_dir / CHANGELOG_FILE).read_text(encoding="utf-8") + return f"v{source_version}", extract_release_body(changelog, source_version) -def get_tag_and_body(): - msg = sys.argv[1] - print(f'msg: {msg}') - p = re.compile('(.*?): ?(.*)') - match = p.search(msg) - assert match is not None, f'commit message format is wrong: {msg}' - tag, body = match[1], match[2] - return body, tag +def add_output(key: str, value: str, output_path: Optional[str] = None) -> None: + output_path = output_path or os.environ.get("GITHUB_OUTPUT") + if output_path is None: + print(f"{key}={value}") + return + with Path(output_path).open("a", encoding="utf-8") as output_file: + output_file.write(f"{key}={value}\n") -def main(): - body, tag = get_tag_and_body() - add_output('tag', tag) +def main(commit_message: Optional[str] = None) -> int: + try: + tag, body = build_release_metadata(commit_message) + (ROOT_DIR / RELEASE_BODY_FILE).write_text(f"{body}\n", encoding="utf-8") + add_output("tag", tag) + except (OSError, SyntaxError, ValueError) as exc: + print(f"Release metadata error: {exc}", file=sys.stderr) + return 1 - with open('release_body.txt', 'w', encoding='utf-8') as f: - f.write(parse_body(body)) + print(f"Release version: {tag.removeprefix('v')}") + print(f"Changelog entries: {count_release_entries(body)}") + print(f"Release body source: {CHANGELOG_FILE}") + return 0 -main() +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1] if len(sys.argv) >= 2 else None)) diff --git a/requirements-dev.txt b/.github/requirements-dev.txt similarity index 94% rename from requirements-dev.txt rename to .github/requirements-dev.txt index f079b387c..223e02fa7 100644 --- a/requirements-dev.txt +++ b/.github/requirements-dev.txt @@ -5,6 +5,7 @@ Pillow psutil pycryptodome requests +rich jm-view-server zhconv img2pdf diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b2cbc42f6..f2b7bbc99 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements-dev.txt + pip install -r .github/requirements-dev.txt pip install -e . - name: Run Performance Benchmark diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml index 5c55240fa..9f461c69b 100644 --- a/.github/workflows/download.yml +++ b/.github/workflows/download.yml @@ -38,7 +38,7 @@ jobs: - name: Install Dependency run: | python -m pip install --upgrade pip - pip install -r requirements-dev.txt + pip install -r .github/requirements-dev.txt - name: 安装jmcomic（pip） if: ${{ github.ref != 'refs/heads/dev' }} diff --git a/.github/workflows/download_dispatch.yml b/.github/workflows/download_dispatch.yml index 09b1637cf..3ab4f5dbd 100644 --- a/.github/workflows/download_dispatch.yml +++ b/.github/workflows/download_dispatch.yml @@ -117,7 +117,7 @@ jobs: - name: Install Dependency run: | python -m pip install --upgrade pip - pip install -r requirements-dev.txt + pip install -r .github/requirements-dev.txt - name: Install img2pdf if: inputs.PDF_OPTION != '否' diff --git a/.github/workflows/export_favorites.yml b/.github/workflows/export_favorites.yml index a37102951..a00cdf64c 100644 --- a/.github/workflows/export_favorites.yml +++ b/.github/workflows/export_favorites.yml @@ -57,7 +57,7 @@ jobs: - name: Install Dependency run: | python -m pip install --upgrade pip - pip install -r requirements-dev.txt + pip install -r .github/requirements-dev.txt sudo apt update sudo apt install p7zip-full diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d9b4b1ce3..8f17f7722 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,15 +1,17 @@ -# publish when new release -name: Publish If Release +# manually release and publish from master +name: Manual Release & Publish on: workflow_dispatch: - release: - types: [ published ] jobs: release: - name: Publish `jmcomic` to PYPI + name: Manually release `jmcomic` + if: github.ref_name == 'master' runs-on: ubuntu-latest + permissions: + id-token: write + contents: write steps: - uses: actions/checkout@v4 @@ -18,12 +20,23 @@ jobs: with: python-version: "3.11" + - name: Build Release Metadata + id: tb + run: | + python .github/release.py + - name: Build run: | python -m pip install build python -m build - - name: Publish PYPI - uses: pypa/gh-action-pypi-publish@release/v1 + - name: Create Release + uses: softprops/action-gh-release@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - password: ${{ secrets.PYPI_JMCOMIC }} + tag_name: ${{ steps.tb.outputs.tag }} + body_path: release_body.txt + + - name: Release PYPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml index 3b85c38c7..0ef2d6cb1 100644 --- a/.github/workflows/release_auto.yml +++ b/.github/workflows/release_auto.yml @@ -2,7 +2,6 @@ name: Auto Release & Publish on: - workflow_dispatch: push: branches: - master @@ -22,12 +21,17 @@ jobs: with: python-version: "3.11" - - name: Parse Tag & Body + - name: Build Release Metadata id: tb run: | commit_message=$(git log --format=%B -n 1 ${{ github.sha }}) python .github/release.py "$commit_message" + - name: Build + run: | + python -m pip install build + python -m build + - name: Create Release uses: softprops/action-gh-release@v2 env: @@ -35,12 +39,6 @@ jobs: with: tag_name: ${{ steps.tb.outputs.tag }} body_path: release_body.txt - generate_release_notes: true - - - name: Build - run: | - python -m pip install build - python -m build - name: Release PYPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test_api.yml b/.github/workflows/test_api.yml index 8caaf5aaa..7f109cfbd 100644 --- a/.github/workflows/test_api.yml +++ b/.github/workflows/test_api.yml @@ -9,6 +9,7 @@ on: paths: - 'src/**/*.py' - 'tests/**/*.py' + - '.github/requirements-dev.txt' - '.github/workflows/test_api.yml' - 'assets/option/option_test_api.yml' @@ -36,7 +37,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + if [ -f .github/requirements-dev.txt ]; then pip install -r .github/requirements-dev.txt; fi - name: Install local run: | diff --git a/.github/workflows/test_html.yml b/.github/workflows/test_html.yml index 56ee60055..9e7bb049e 100644 --- a/.github/workflows/test_html.yml +++ b/.github/workflows/test_html.yml @@ -9,6 +9,7 @@ on: paths: - 'src/**/*.py' - 'tests/**/*.py' + - '.github/requirements-dev.txt' - '.github/workflows/test_html.yml' - 'assets/option/option_test_html.yml' @@ -36,7 +37,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + if [ -f .github/requirements-dev.txt ]; then pip install -r .github/requirements-dev.txt; fi - name: Install local run: | diff --git a/.gitignore b/.gitignore index 65e54d52d..80f5a7101 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # this repo /assets/download/ +/docs/superpowers/** # Byte-compiled / optimized / DLL files @@ -163,4 +164,4 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ .agent -AGENTS.md \ No newline at end of file +AGENTS.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..b5e8e9b80 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,82 @@ +# Changelog + +本文件记录 jmcomic 的版本变化。2.7.0 之前的更新记录请见 GitHub Releases。 + +条目分类参考 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.0.0/)， +版本号遵循 [语义化版本](https://semver.org/lang/zh-CN/)。 + +## [2.7.4] - 2026-08-09 + +### Summary + +本次更新让同步和异步下载进度更直观、下载结果更易处理，还可以分页浏览全站评论并直接获取当前页码。 + +### Added +- 新增 `download_progress` 插件：`jmcomic` 命令安装 `rich` 后默认启用，可通过 `--no-progress` 关闭自动启用；同步和异步下载在交互终端中固定展示最近 6 条日志，并在下方显示彩色的本子、章节两级进度；IDE 运行面板等不支持动态界面的环境只在结束后输出汇总；完整日志写入文件且不再重复输出 `INFO` 字段。 + +![download_progress 插件效果](https://raw.githubusercontent.com/hect0x7/JMComic-Crawler-Python/v2.7.4/assets/docs/sources/images/download_progress_terminal.png) + +- 新增下载清单 `DownloadManifest`，聚合一次顶层下载产生的图片路径和导出文件。 +- `DownloadResult` 新增 `manifest` 和 `duration` 属性，同时保持原有二元组解包兼容性。 +- 新增获取全站评论分页及生成器 API，支持 HTML、API 和异步 API 客户端。 +- 搜索、分类、收藏夹和评论分页结果新增 `page_number` 字段，可直接获取当前页码。 +- GitHub Actions 下载支持压缩图片，减少最终压缩包体积。 + +### Changed +- `JmDownloader.use()` 和 `JmAsyncDownloader.use()` 现在会记录替换前后的 Downloader class，异步下载 API 也会使用已替换的默认异步 Downloader。 +- 同步与异步下载流程统一记录成功下载及缓存命中的图片，并使用图片插件处理后的最终保存路径。 +- `result.duration` 记录从调用下载 API 到返回所用的总时间；本子、章节和图片实体分别记录各自的下载耗时。 +- Feature 根据当前 `TaskContext` 判断顶层下载类型，不再需要额外传入 `feature_from`。 +- ZIP、PDF 和长图插件产物按文件后缀登记到下载清单。 +- 详情缓存返回独立干净副本，避免下载路径、耗时等状态污染后续缓存结果。 +- `JmAlbumComment` 新增可读的字符串输出，支持直接打印评论。 +- 完善 HTML 评论解析，补齐全站评论中的本子 ID 和用户 ID。 +- 全站评论生成器分别使用 API 总页数和 HTML 重复页判断结束，避免两类响应混用同一停止条件。 +- 使用 `--no-progress` 但 Option 已配置 `download_progress` 时，在固定启动面板中显示冲突提醒。 +- GitHub Release 改为从对应版本的 `CHANGELOG.md` 生成发布说明，并支持手动触发发布流程。 +- 重写下载返回值与异步下载文档，统一使用同时兼容 GitHub 和 MkDocs 的提示及折叠语法。 +- 综合插件示例并入插件教程，移除使用价值较低的“模块自定义”教程，并将日志接管方式迁移到日志教程。 + +### Removed +- 按照弃用计划移除 `jmcomic.cl` 兼容模块，请使用 `jmcomic.cli`。 + +## [2.7.3] - 2026-08-03 + +### Added +- 新增获取本子评论的 API。 +- 新增任务上下文，可按任务维度标记、传播和收集日志。 + +### Changed +- 精简下载 API 参数。 +- 兼容 `jm-view-server` 项目改名。 +- 完善任务日志及下载 API 使用文档。 + +## [2.7.2] - 2026-07-16 + +### Added +- 异步 API 新增 `categories_filter_gen`。 + +### Changed +- 更新 JM 内置域名。 +- 完善异步 API、测试和异步使用文档。 + +### Removed +- 移除已经失效的 GitHub 域名抓取实现。 + +## [2.7.1] - 2026-07-04 + +### Changed +- 优化异步请求的重试配置。 +- 调整批量下载返回值类型。 +- `IndexedEntity` 继承 `Sequence`，支持标准切片协议。 +- 更新 README 导览图。 + +## [2.7.0] - 2026-06-15 + +### Added +- 新增完整的异步 Client、Downloader 和下载 API。 +- HTML 正则解析失败时自动保存网页内容，方便定位解析问题。 +- 新增同步与异步下载性能 Benchmark。 + +### Changed +- 补充异步 API 相关文档并优化既有文档。 diff --git a/README.md b/README.md index eebf0bbcf..df0dfd41b 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,16 @@ jmcomic 123 jmcomic 123 p456 ``` +安装 `rich` 后，`jmcomic` 命令会默认显示本子和章节下载进度： + +```sh +pip install rich +``` + +![jmcomic 命令行下载进度](./assets/docs/sources/images/download_progress_terminal.png) + +不想显示进度条时，可以使用 `jmcomic 123 --no-progress`。没有安装 `rich` 时会自动使用普通日志。 + 命令行模式也支持自定义option，你可以使用环境变量或者命令行参数： a. 通过命令行--option参数指定option文件路径 diff --git a/assets/docs/mkdocs.yml b/assets/docs/mkdocs.yml index 744088078..45b2f838d 100644 --- a/assets/docs/mkdocs.yml +++ b/assets/docs/mkdocs.yml @@ -49,10 +49,8 @@ nav: - tutorial/0_common_usage.md - tutorial/1_github_actions.md - tutorial/2_command_line.md - - tutorial/4_module_custom.md - tutorial/5_filter.md - tutorial/6_plugin.md - - tutorial/7_advance.md - tutorial/8_pick_domain.md - tutorial/9_custom_download_dir_name.md - tutorial/10_export_favorites.md @@ -78,6 +76,8 @@ plugins: markdown_extensions: - attr_list + - admonition + - md_in_html - toc: permalink: true - pymdownx.highlight @@ -92,4 +92,4 @@ markdown_extensions: - pymdownx.snippets - pymdownx.superfences -docs_dir: sources \ No newline at end of file +docs_dir: sources diff --git a/assets/docs/sources/images/download_progress_terminal.png b/assets/docs/sources/images/download_progress_terminal.png new file mode 100644 index 000000000..9ebb87377 Binary files /dev/null and b/assets/docs/sources/images/download_progress_terminal.png differ diff --git a/assets/docs/sources/images/github_actions_tutorial.jpg b/assets/docs/sources/images/github_actions_tutorial.jpg index 88bc25064..d325f0af1 100644 Binary files a/assets/docs/sources/images/github_actions_tutorial.jpg and b/assets/docs/sources/images/github_actions_tutorial.jpg differ diff --git a/assets/docs/sources/index.md b/assets/docs/sources/index.md index 8d06270fd..0ca28e281 100644 --- a/assets/docs/sources/index.md +++ b/assets/docs/sources/index.md @@ -17,6 +17,7 @@ - [快速上手(GitHub README)](https://github.com/hect0x7/JMComic-Crawler-Python/tree/master?tab=readme-ov-file#%E5%BF%AB%E9%80%9F%E4%B8%8A%E6%89%8B) - [常用类和方法演示](tutorial/0_common_usage.md) +- [下载进度条](tutorial/15_download_progress.md) - [Async API用法](tutorial/14_async_usage.md) - [下载后转为 PDF / ZIP / 长图](tutorial/13_export_and_feature.md) - [option配置以及插件写法](./option_file_syntax.md) @@ -38,7 +39,6 @@ - [下载文件夹名](tutorial/9_custom_download_dir_name.md) - [日志](tutorial/11_log_custom.md) -- [模块](tutorial/4_module_custom.md) ## 相关项目 diff --git a/assets/docs/sources/tutorial/0_common_usage.md b/assets/docs/sources/tutorial/0_common_usage.md index 51e31a3d4..dcabaf402 100644 --- a/assets/docs/sources/tutorial/0_common_usage.md +++ b/assets/docs/sources/tutorial/0_common_usage.md @@ -13,6 +13,12 @@ download_photo(438696) # 同时下载多个本子 download_album([123, 456, 789]) + +# 查看本子/章节下载位置和耗时 +result: DownloadResult = download_album(123) +album: JmAlbumDetail = result.detail # detail是实体类，download_album 返回 album，download_photo 返回 photo +print(f'本子-JM{album.id}，下载保存文件夹: {album.save_path}, 下载耗时: {result.duration:.3f}秒') +# DownloadResult 里还包含大量字段，更多用法请查阅下方章节【下载返回值】 ``` ## 使用option定制化下载本子 @@ -170,45 +176,64 @@ for aid, atitle, tag_list in page.iter_id_title_tag(): # 使用page的iter_id_t download_album(aid_list, option) ``` -## 获取本子评论 +## 获取评论 -```python -from jmcomic import JmOption, JmAlbumCommentPage, JmAlbumComment +目前支持获取两种评论： -client = JmOption.default().new_jm_client(impl='api') +| 评论类型 | 说明 | 获取一页的方法 | 自动翻页的方法 | +| --- | --- | --- | --- | +| 本子评论 | 获取指定本子下的评论和回评 | `album_pagination` | `album_pagination_gen` | +| 全站评论 | 获取全站最新发布的评论，可通过 `comment.album_id` 知道评论来自哪个本子 | `forum_pagination` | `forum_pagination_gen` | -# 获取第一页评论 -page: JmAlbumCommentPage = client.album_pagination('123456') +评论页可以直接遍历，评论对象也可以直接打印，格式为 `[评论ID] 用户（剧透）: 内容`。 -print('本页评论数:', len(page)) -print('本页评论数（含回评）:', page.comment_count) -print('本子的评论总数:', page.total) -print('总页数:', page.page_count) +### 获取本子评论 -# page对象可以直接遍历评论，评论类型是JmAlbumComment -comment: JmAlbumComment -for comment in page: - print('用户:', comment.nickname or comment.username) - print('内容:', comment.content) - print('是否剧透:', comment.is_spoiler) +`album_pagination` 获取指定本子的一页评论。 + +```python +from jmcomic import JmOption, JmAlbumComment + +client = JmOption.default().new_jm_client(impl='api') +page = client.album_pagination('123456') - # 回评也是评论对象 +print(f'第 {page.page_number}/{page.page_count} 页，当前页一共 {len(page)} 条主评论，整本一共 {page.total} 条主评论') +for comment in page: + comment: JmAlbumComment # 评论是实体类 + print( + f'评论ID: {comment.comment_id} | 用户ID: {comment.user_id} | 用户: {comment.nickname or comment.username} | ' + f'是否剧透: {comment.is_spoiler} | 点赞数: {comment.likes} | 发布时间: {comment.created_at}\n内容: {comment.content}' + ) for reply in comment.replies: - print('回评用户:', reply.nickname or reply.username) - print('回评内容:', reply.content) - print('回评是否剧透:', reply.is_spoiler) + reply: JmAlbumComment # 回评也是相同的实体类 + print(' └─', reply) -# gen 方法支持自动循环获取评论分页，直到结束 +# 需要连续获取分页时，使用生成器： for page in client.album_pagination_gen('123456'): - print('本页主评论数:', len(page)) - print('本页评论数（含回评）:', page.comment_count) - print('主评论总数:', page.total) - print('总页数:', page.page_count) + print(f'\n第 {page.page_number} 页') + for comment in page: + print(comment) +``` + +### 获取全站评论 + +`forum_pagination` 获取一页全站评论。 + +> [!NOTE] +> html 端不提供 `total`（全部分页的主评论总数）和 `page_count`（总页数），这两个字段都为 `None`，api端则有值。 + +```python +page = client.forum_pagination(page=1) + +print(f'第 {page.page_number}/{page.page_count} 页，当前页一共 {len(page)} 条主评论，全站一共 {page.total} 条主评论') +for comment in page: + print(f'本子 {comment.album_id} | {comment}') +# 全站评论同样支持生成器 +for page in client.forum_pagination_gen(page=1): + print(f'\n第 {page.page_number} 页') for comment in page: - print('用户:', comment.nickname or comment.username) - print('内容:', comment.content) - print('是否剧透:', comment.is_spoiler) + print(f'本子 {comment.album_id} | {comment}') ``` ## 获取收藏夹 @@ -413,3 +438,170 @@ cl = JmApiClient( retry_times=1 ) ``` + + +## 下载返回值 + +`download_album` 和 `download_photo` 下载完成后，单个 ID 返回 `DownloadResult`，多个 ID 返回 `BatchResult`。 + +从 `result.detail` 可以取得下载的本子/章节的实体类： + +```python +from jmcomic import download_album, download_photo + +# 下载本子；result.detail 是本子实体 +result = download_album('123') +album = result.detail +print(f'本子实体: {album}') + +# 下载章节；result.detail 是章节实体 +result = download_photo('456') +photo = result.detail +print(f'章节实体: {photo}') +``` + +### 获取保存路径和耗时 + +下载完成后，你通常会关心两件事：文件保存在哪里，以及哪一步比较慢。所有 `duration` 的单位都是秒。 + +| 对象 | `save_path` | `duration` | +| --- | --- |------------------------------------------------------| +| 下载结果 `result` | 通过 `result.detail.save_path` 查看 | 从调用下载方法到返回，你总共等了多久 | +| 本子 `album` | 本子根目录 | 下载这个本子花了多久，包含获取详情、下载章节和插件 | +| 章节 `photo` | 章节图片目录 | 下载这个章节花了多久，包含获取详情、下载图片和插件 | +| 图片 `image` | 图片文件路径 | 下载这张图片花了多久，包含检查缓存、下载、保存和插件 | + +> 一次本子下载可能同时处理多个章节，一个章节也可能同时处理多张图片。因此，把所有章节或图片的耗时相加，不会得到本子的耗时，这是正常现象。 + + +
+完整示例：查看路径、耗时和缓存状态 + +```python +from jmcomic import download_album + +result = download_album('123') +album = result.detail + +# result.duration 是调用下载方法后总共等待的时间 +print(f'本子目录: {album.save_path}，总共等待: {result.duration:.2f} 秒') +print(f'下载本子用了: {album.duration:.2f} 秒') + +for photo in album: + # 查看每个章节的保存目录和处理耗时 + print(f'章节 {photo.id} 目录: {photo.save_path}，耗时: {photo.duration:.2f} 秒') + + for image in photo: + # exists 和 cache 都为 True，表示满足缓存复用条件 + not_download = image.exists and image.cache + print( + f'图片 {image.filename} 路径: {image.save_path}，' + f'耗时: {image.duration:.2f} 秒，是否因存在而跳过下载: {not_download}' + ) +``` + +`not_download` 为 `True`，表示目标图片原本存在并且允许使用缓存。 + +
+ +### 使用 `manifest` 获取结果文件 + +相比于遍历实体类， `manifest` 提供了更直接的写法，适合场景：直接取得全部图片路径，以及取得**额外产物**（由插件/Feature产出的 PDF、ZIP、长图） + +| 想要什么 | 推荐写法 | +|--------------------------------------------| --- | +| 查看某张图片的路径、耗时和缓存状态 | 遍历实体，读取 `image.save_path` 等字段 | +| 直接获取所有图片路径（包含命中缓存的图片） | 用`manifest`，`result.manifest.image_filepath_list` | +| 额外产物（PDF、ZIP 或长图） | 用`manifest`，`result.manifest.get_export_filepath_list('文件后缀')` | + +
+完整示例：获取图片和导出文件 + +```python +from jmcomic import Feature, download_album + +# 下载本子，并使用内置 Feature 导出 PDF、ZIP 和长图 +result = download_album( + '123', + extra=Feature.export_pdf + Feature.export_zip + Feature.export_long_img, +) + +# 本次成功下载或直接复用的图片路径 +print('图片文件:', result.manifest.image_filepath_list) + +# 插件导出的文件按后缀查询，后缀前面的点可以省略 +pdf_filepath_list = result.manifest.get_export_filepath_list('pdf') +zip_filepath_list = result.manifest.get_export_filepath_list('.zip') +png_filepath_list = result.manifest.get_export_filepath_list('png') + +print('PDF 文件:', pdf_filepath_list) +print('ZIP 文件:', zip_filepath_list) +print('长图导出文件:', png_filepath_list) +``` + +
+ +### 批量下载的返回值 + +传入多个 ID 时，返回值是 `BatchResult`。每一项成功下载对应一个 `DownloadResult`，失败任务则记录在 `failed` 中： + +
+完整示例：处理批量下载结果 + +```python +from jmcomic import download_album + +# 同时下载多个本子 +batch_result = download_album(['123', '456', '789']) + +# BatchResult 继承 set，成功结果没有输入顺序保证 +for result in batch_result: + album = result.detail + # 通过实体 ID 识别当前结果，不要用遍历位置对应输入列表 + print(f'JM{album.id} 下载到: {album.save_path}') + +# failed 的键是下载失败的 ID，值是记录失败原因的异常对象 +for album_id, error in batch_result.failed.items(): + print(f'JM{album_id} 下载失败: {error}') + +# total 是实际处理的不同 ID 数量，重复 ID 不会重复下载 +print('任务总数:', batch_result.total) +print('是否全部成功:', batch_result.all_succeeded) +``` + +
+ +下载单个 ID 时，请求本子失败会直接抛出异常；如果只有部分章节或图片失败，会在任务结束后汇总抛出 `PartialDownloadFailedException`，此时不会返回 `DownloadResult`。批量下载则继续执行其他任务，并把失败项集中放进 `batch_result.failed`。 + +### 速查表 + +| 你的需求 | 推荐写法 | +| --- | --- | +| 查看本子或章节信息 | `result.detail` | +| 查看本子或章节目录 | `result.detail.save_path` | +| 查看单张图片路径和状态 | 遍历实体后读取 `image.save_path` 等字段 | +| 查看图片或章节失败原因 | 捕获 `PartialDownloadFailedException` 后，读取 `e.downloader.download_failed_image` / `download_failed_photo`；列表元素为 `(实体, 异常)` | +| 获取本次成功图片路径列表 | `result.manifest.image_filepath_list` | +| 获取已登记的导出文件路径 | `result.manifest.get_export_filepath_list('后缀')` | +| 查看单个 ID 下载的完整时间 | `result.duration` | +| 定位本子、章节或图片的内部处理慢点 | 对应实体的 `duration` | +| 检查批量下载失败项 | `batch_result.failed` | + +
+兼容旧版本的返回值解包写法 + +旧代码可能会把返回值直接解包成两个变量，这种写法仍然可以继续使用： + +```python +from jmcomic import download_album + +result = download_album('123') + +# 旧写法：第一个变量是本子实体，第二个变量是下载器 +album, downloader = result + +# 新代码更推荐直接通过 result.detail 读取本子实体 +assert album is result.detail +``` + +
diff --git a/assets/docs/sources/tutorial/11_log_custom.md b/assets/docs/sources/tutorial/11_log_custom.md index 9b6e893a0..7c60ba269 100644 --- a/assets/docs/sources/tutorial/11_log_custom.md +++ b/assets/docs/sources/tutorial/11_log_custom.md @@ -147,12 +147,44 @@ with jm_task_context(task_id=task_id): # 任务id 根据你的需求复杂度，你可以选择以下方式： -- **方式 A：操作 jm_logger (推荐 / 标准)** +| 方式 | 适用场景 | 推荐程度 | +| --- | --- | --- | +| 操作 `jm_logger` | 修改输出位置、格式和过滤规则，或者接入标准日志系统 | 推荐 | +| 接管 `JmModuleConfig.EXECUTOR_LOG` | 完全替换日志分发逻辑，或者接入不兼容 `logging` 的系统 | 仅限特殊需求 | - 适用于：改变日志输出位置（如文件、监控、后端服务）、调整显示格式、自定义过滤。 +### 5.1 操作 `jm_logger` -- **方式 B：接管 EXECUTOR_LOG (高级 / 深度定制)** +jmcomic 使用名为 `jmcomic` 的标准 Python Logger。下面的示例会移除默认输出方式，并将日志写入文件： - 适用于：需要完全重塑日志的分发逻辑，或者将日志直接桥接到不符合标准 logging 协议的第三方系统。 +```python +import logging + +from jmcomic import jm_logger + + +jm_logger.handlers.clear() +jm_logger.addHandler( + logging.FileHandler('jm_download.log', encoding='utf-8') +) +``` + +`handlers.clear()` 会同时移除默认的终端输出。如果希望日志同时显示在终端并写入文件，只需要保留原有 Handler，再追加新的 `FileHandler`。 + +### 5.2 接管 `EXECUTOR_LOG` + +只有标准 `logging` 无法满足需求时，才建议替换 `EXECUTOR_LOG`： + +```python +from jmcomic import JmModuleConfig + + +def custom_log(topic: str, msg, error: Exception = None): + # topic 是日志主题，例如 api、album.after、plugin.error + # msg 是日志内容；error 是可选的异常对象 + print(topic, msg, error) + + +JmModuleConfig.EXECUTOR_LOG = custom_log +``` -代码示例：[模块自定义-自定义log](./4_module_custom.md#自定义log) +这是全局设置，会影响当前进程中之后产生的全部 jmcomic 日志。普通的文件输出、日志过滤和格式调整，应优先操作 `jm_logger`。 diff --git a/assets/docs/sources/tutorial/12_domain_strategy.md b/assets/docs/sources/tutorial/12_domain_strategy.md index 5da3d52a9..257a522aa 100644 --- a/assets/docs/sources/tutorial/12_domain_strategy.md +++ b/assets/docs/sources/tutorial/12_domain_strategy.md @@ -77,6 +77,7 @@ op.client = op.new_jm_client(domain_list=[domain], impl='html') 默认的机制是在单次请求报错时，按顺序尝试数组内的下一个域名。 如果经常遇到连接断开或超时，可以使用 `advanced_retry` 插件。该插件提供： + - 记录历史失败次数 - 限制单个域名的最大失败次数（超过则拉黑废弃） - 对列表循环多轮尝试等容错机制 diff --git a/assets/docs/sources/tutorial/13_export_and_feature.md b/assets/docs/sources/tutorial/13_export_and_feature.md index 3829c5da9..bdaf422cd 100644 --- a/assets/docs/sources/tutorial/13_export_and_feature.md +++ b/assets/docs/sources/tutorial/13_export_and_feature.md @@ -3,6 +3,7 @@ ## 1. 需求场景 下载本子后，很多用户有进一步导出的需求： + - 导出为 **PDF**：方便在电子阅读器上查看 - 导出为 **ZIP**：方便传输和存档 - 合并为 **长图**：方便一张图看完整个章节 @@ -111,7 +112,7 @@ download_photo('456', option, extra=Feature.export_pdf) ├── [JM{Pid}]章节标题.pdf ← 该章节导出为 1 个 PDF ``` -> 💡 **提示**：同一个 Feature，通过 `download_album` 和 `download_photo` 调用时会自动适配不同的导出行为，详见下方 [智能适配规则](#25-智能适配规则)。 +> 💡 **提示**：同一个 Feature，通过 `download_album` 和 `download_photo` 调用时会自动适配不同的导出行为，详见下方 [智能适配规则](#25)。 ### 2.5 智能适配规则 @@ -124,7 +125,7 @@ download_photo('456', option, extra=Feature.export_pdf) 当你显式传入参数时（如 `filename_rule='Ptitle'`），**你的配置优先**，不会被自适应覆盖。 -> 💡 **提示**：更多可选参数（如加密密码 `encrypt`、后缀名 `suffix` 等），参考 [Plugin 插件参数大全](../option_file_syntax.md#3-option插件配置项)。 +> 💡 **提示**：更多可选参数（如加密密码 `encrypt`、后缀名 `suffix` 等），参考 [Plugin 插件参数大全](../option_file_syntax.md#3-option)。 ## 3. 传统写法（YAML 插件配置） @@ -176,18 +177,18 @@ api.download_album(extra=Feature.export_pdf) │ ├→ download jmcomic images ... # 下载禁漫图片 │ └→ after_photo(photo) │ └→ _invoke_features_for('after_photo') - │ └→ pdf.should_invoke('after_photo', 'download_album') → False ✗ 跳过 + │ └→ pdf.should_invoke('after_photo') → False ✗ 跳过 │ └→ after_album(album) └→ _invoke_features_for('after_album') - └→ pdf.should_invoke('after_album', 'download_album') → True ✓ 执行! - └→ _adapt_plugin_kwargs(from, when) # 动态生成插件参数 + └→ pdf.should_invoke('after_album') → True ✓ 执行! + └→ _adapt_plugin_kwargs(option, when) # 动态生成插件参数 └→ option.invoke(pdf, kwargs) # 调用pdf插件，传入参数 ``` > 💡 **关键点**： > -> - **执行时机**：`PluginFeature` 根据注册来源自动推导（`download_album` → `after_album`，`download_photo` → `after_photo`）。自定义 Feature 默认在所有事件都会执行，你可以覆写 `should_invoke` 来控制。 +> - **执行时机**：`PluginFeature` 根据 `TaskContext` 中的 `download_type` 判断当前顶层下载类型。自定义 Feature 默认在所有事件都会执行，你可以覆写 `should_invoke` 来控制。 > - **参数自适应**：`PluginFeature` 的 `filename_rule` 前缀（A/P）会根据来源动态适配。ZIP 的打包粒度由插件根据上下文自动推导。用户显式传入的参数不会被覆盖。 ### 自定义 Feature diff --git a/assets/docs/sources/tutorial/14_async_usage.md b/assets/docs/sources/tutorial/14_async_usage.md index 7e062b542..d2e08702b 100644 --- a/assets/docs/sources/tutorial/14_async_usage.md +++ b/assets/docs/sources/tutorial/14_async_usage.md @@ -55,24 +55,14 @@ asyncio.run(main()) ### 💡 关于 async with 和自动初始化 -当你使用异步客户端时，推荐直接搭配 `async with` 上下文管理器来使用： +当你使用异步client时，推荐直接搭配 `async with` 上下文管理器来使用： ```python -# 离开代码块时会自动清理并断开连接 +# 推荐用法，离开代码块时会自动清理并断开连接 async with JmOption.default().new_jm_async_client() as cl: album = await cl.get_album_detail(123) -``` - -客户端会在你真正发起网络请求时自动初始化： - -- **结合 `async with`**：当进入 `async with` 作用域时，客户端会自动完成域名解析、联通性检查等必要的初始化工作，并在离开时安全释放连接。 -- **单独使用**：如果你不想使用 `async with`，而是直接调用 `cl = op.new_jm_async_client()`，那么在第一次发起真实的请求（比如 `get_album_detail`）时，客户端也会自动检测并先执行一遍初始化。 - -无论哪种写法都只会初始化一次，你不需要自己去调用任何初始化代码，直接用就行。 -如果不使用 `async with`，使用完成后需要显式关闭客户端： - -```python +# 另外一种用法，单独使用client cl = JmOption.default().new_jm_async_client() try: album = await cl.get_album_detail(123) @@ -80,6 +70,19 @@ finally: await cl.close() ``` +`async with` 和单独使用client都会自动初始化，区别主要在初始化时机和生命周期管理： + +| 对比项 | 使用 `async with` | 单独使用client | +| --- | --- |-------------------------------------------| +| 初始化时机 | 进入 `async with` 作用域时自动初始化 | 第一次发起真实请求前自动初始化 | +| 初始化内容 | 自动完成域名解析、联通性检查等必要工作 | 同左 | +| 初始化次数 | 只初始化一次 | 同左 | +| 连接清理 | 离开作用域时自动安全释放连接 | 使用完成后必须显式调用 `await cl.close()` | +| 推荐场景 | 推荐用于一般请求和下载任务 | 需要自行控制客户端生命周期时使用 | + +两种写法都不需要手动调用初始化方法；如果单独使用客户端，请通过 `try/finally` 确保连接一定会被关闭。 + + --- ### 并发请求示例 @@ -104,10 +107,6 @@ async def main(): # 打印结果 for aid, album in zip(album_id_list, album_list): print(f'[JM{aid}] 本子详情: {album}') - - # 获取章节实体类 - photo = await cl.get_photo_detail('212214') - print(photo.name) asyncio.run(main()) ``` @@ -148,7 +147,7 @@ async def main(): async with JmOption.default().new_jm_async_client() as cl: # async for 会帮你自动加载下一页，一页一页往下搜 async for page in cl.search_gen('+MANA +无修正'): - print(f'当前获取到了第 {page.page} 页，本页数据量: {page.page_size}') + print(f'当前获取到了第 {page.page_number} 页，本页数据量: {len(page)}，总数量: {page.total}') for album_id, title in page.iter_id_title(): print(f'[{album_id}]: {title}') @@ -211,7 +210,7 @@ async def main(): category=JmMagicConstants.CATEGORY_ALL, order_by=JmMagicConstants.ORDER_BY_VIEW, ): - print(page.page) + print(f'当前获取到了第 {page.page_number} 页，本页数据量: {len(page)}') asyncio.run(main()) ``` @@ -229,3 +228,39 @@ client: # 指定异步客户端的底层实现类 (目前仅有: async_api) async_impl: async_api ``` + +## 9. 查看下载耗时 + +异步下载完成后，可以直接查看自己总共等了多久，也可以继续查看具体是哪个本子、章节或图片比较慢。所有 `duration` 的单位都是秒。 + +```python +import asyncio +import jmcomic + + +async def main(): + result = await jmcomic.download_album_async('438696') + album = result.detail + + # 从调用 download_album_async 到返回，总共等了多久 + print(f'总共等待: {result.duration:.3f} 秒') + + # 如果下载比较慢，可以继续查看具体慢在哪里 + print(f'下载本子用了: {album.duration:.3f} 秒') + for photo in album: + print(f'下载章节 {photo.id} 用了: {photo.duration:.3f} 秒') + for image in photo: + print(f'处理图片 {image.img_file_name} 用了: {image.duration:.3f} 秒') + + +asyncio.run(main()) +``` + +| 字段 | 它告诉你什么 | +| --- | --- | +| `result.duration` | 从调用异步下载方法到返回，你总共等了多久 | +| `album.duration` | 下载这个本子花了多久，包含获取本子信息和整理下载结果 | +| `photo.duration` | 下载这个章节花了多久，包含获取或补全章节信息 | +| `image.duration` | 处理这张图片花了多久，包含检查缓存、下载、解密和保存 | + +下载器可能同时处理多个章节或多张图片，所以把它们的耗时全部相加，不会得到本子的耗时，这是正常现象。同步下载中的这些字段含义相同。 diff --git a/assets/docs/sources/tutorial/15_download_progress.md b/assets/docs/sources/tutorial/15_download_progress.md new file mode 100644 index 000000000..ed1ea36ec --- /dev/null +++ b/assets/docs/sources/tutorial/15_download_progress.md @@ -0,0 +1,82 @@ +# 下载进度展示（插件） + +使用 `download_album` 下载多章节本子时，默认日志会不断换行，很难一眼看出整本和各章节的下载进度。 + +自 `v2.7.4` 起，jmcomic 内置了美观的下载进度条，效果如下： + +![下载进度插件的终端效果](../images/download_progress_terminal.png) + +使用 `jmcomic` 命令下载时，安装 `rich` 后会自动开启进度条；使用 `--no-progress` 可以关闭自动启用。通过 Python API 下载时，配置 `download_progress` 插件即可开启。 + +插件会自动完成这些工作： + +- 同时支持普通下载和异步下载。 +- 在进度条上方固定显示最近 6 条 jmcomic 日志。 +- 在进度条下方显示本子、章节两级下载进度。 +- 本子进度 = 已下载章节 / 总章节；章节进度 = 已下载图片 / 总图片。 +- 将完整日志写入 `jmcomic-download.log`。 + +## 为什么有些窗口不显示动态进度 + +动态进度条不是不断打印新行，而是反复覆盖终端中的同一块区域。PowerShell、Windows Terminal 等终端支持这种刷新方式，因此可以让日志固定在上方、进度条固定在下方。 + +PyCharm 的 **Run 运行面板**以及部分 IDE 输出窗口会把程序输出当成普通文本，无法正确覆盖旧内容。如果强行刷新，每一帧都会变成新的一行，最终造成刷屏。 + +因此，插件会自动判断当前窗口是否支持动态刷新： + +- 支持：显示固定日志区和两级进度条。 +- 不支持：启动时显示一次说明，下载期间不刷新，结束后输出一次本子、章节和图片数量汇总。 + +如果你使用 PyCharm，请在底部的 **Terminal** 中运行 `python script.py`，即可查看动态进度。 + +## 1. 安装 + +使用插件前需要额外安装 `rich`: + +```shell +pip install -U jmcomic rich +``` + +## 2. 使用插件 + +```python +from jmcomic import create_option_by_str, download_album + +option = create_option_by_str(''' +plugins: + after_init: + - plugin: download_progress +''') + +download_album('123456', option) +``` + +异步版本也支持： + +```python +import asyncio + +from jmcomic import create_option_by_str, download_album_async + +option = create_option_by_str(''' +plugins: + after_init: + - plugin: download_progress +''') + +asyncio.run(download_album_async('123456', option)) +``` + +图片和 `jmcomic-download.log` 默认保存在运行命令的当前目录中。重复运行时，新日志会追加到已有日志文件末尾。 + +## 3. 运行时会看到什么 + +| 运行位置 | 下载期间 | 下载结束后 | +| --- | --- | --- | +| Terminal、PowerShell | 上方固定显示最近 6 条日志，下方实时刷新本子和章节进度条 | 进度条停留在完成状态 | +| PyCharm Run 等普通输出窗口 | 不刷新进度，避免重复打印造成刷屏 | 输出一次本子、章节和图片数量汇总 | + +两种模式都会将完整日志追加到 `jmcomic-download.log`。下载入口和并发调度仍由 jmcomic 负责，插件只负责展示进度。 + +> [!NOTE] +> 如果你通过自定义规则跳过了部分章节或图片，或者下载过程中发生失败，进度可能不会到达原始总数。具体原因可以在日志文件中查看。 diff --git a/assets/docs/sources/tutorial/4_module_custom.md b/assets/docs/sources/tutorial/4_module_custom.md deleted file mode 100644 index d804702cc..000000000 --- a/assets/docs/sources/tutorial/4_module_custom.md +++ /dev/null @@ -1,197 +0,0 @@ -# 模块自定义 - - - -## 自定义下载事件的回调函数 - -```python -def custom_download_event_callback(): - """ - 该函数演示如何自定义下载事件的回调函数。 - 公开下载 API 不接收 callback 参数；需要响应下载事件时，请自定义 Downloader。 - """ - - # jmcomic的下载功能由 JmModuleConfig.CLASS_DOWNLOADER 这个类来负责执行 - # 这个类默认是 JmDownloader，继承了 DownloadCallback - # DownloadCallback 定义了 Downloader 内部各类下载事件的回调方法 - # 你可以继承 JmDownloader 并覆盖相应方法来响应事件 - class MyDownloader(JmDownloader): - # 覆盖 album 下载完成事件的回调函数 - def after_album(self, album: JmAlbumDetail): - print(f'album下载完毕: {album}') - pass - - # 最后，让你的自定义类生效 - JmModuleConfig.CLASS_DOWNLOADER = MyDownloader -``` - - - -## 自定义option类 - - -```python -def custom_option_class(): - """ - 该函数演示自定义option类 - """ - - # jmcomic模块支持自定义Option类， - # 你可以写一个自己的类，继承JmOption，然后覆盖其中的一些方法。 - class MyOption(JmOption): - - def __init__(self, *args, **kwargs): - print('MyOption 初始化开始') - super().__init__(*args, **kwargs) - - @classmethod - def default(cls): - print('调用了MyOption.default()') - return super().default() - - # 最后，替换默认Option类即可 - JmModuleConfig.CLASS_OPTION = MyOption -``` - - -## 自定义client类 - -```python -def custom_client_class(): - """ - 该文件演示自定义client类 - """ - - # 默认情况下，JmOption使用client类是根据配置项 `client.impl` 决定的 - # JmOption会根据`client.impl`到 JmModuleConfig.CLASS_CLIENT_IMPL 中查找 - - # 自定义client的步骤如下 - - # 1. 自定义Client类 - class MyClient(JmHtmlClient): - client_key = 'myclient' - pass - - # 2. 让MyClient生效 - JmModuleConfig.register_client(MyClient) - - # 3. 在配置文件中使用你定义的client.impl，后续使用这个option即可 - """ - client: - impl: myclient - """ -``` - - -## 自定义实体类（本子/章节/图片） - -```python -def custom_album_photo_image_detail_class(): - """ - 该函数演示自定义实体类（本子/章节/图片） - - 在使用路径规则 DirRule 时，可能会遇到需要自定义实体类属性的情况，例如： - dir_rule: - base_dir: ${workspace} - rule: Bd_Acustom_Pcustom - # 可选：对目录名进行繁/简体规范化（None/zh-cn/zh-tw），默认不启用 - # normalize_zh: zh-cn - - 上面的Acustom，Pcustom都是自定义字段 - 如果你想要使用这种自定义字段，你就需要替换默认的实体类，方式如下 - """ - - # 自定义本子实体类 - class MyAlbum(JmAlbumDetail): - # 自定义 custom 属性 - @property - def custom(self): - return f'custom_{self.title}' - - # 自定义章节实体类 - class MyPhoto(JmPhotoDetail): - # 自定义 custom 属性 - @property - def custom(self): - return f'custom_{self.title}' - - """ - v2.3.3: 支持更灵活的自定义方式，可以使用函数，效果同上，示例见下 - """ - - class MyAlbum2(JmAlbumDetail): - - def get_dirname(self, ref: str) -> str: - if ref == 'custom': - return f'custom_{self.name}' - - return super().get_dirname(ref) - - # 最后，替换默认实体类来让你的自定义类生效 - JmModuleConfig.CLASS_ALBUM = MyAlbum - JmModuleConfig.CLASS_PHOTO = MyPhoto -``` - - - -## 自定义log - -```python -def custom_jm_log(): - """ - 该函数演示如何接管和自定义日志输出 - """ - - # jmcomic 项目默认使用内置的 Python logging 模块 - # 其日志记录器的名字固定为 "jmcomic" - - # 【推荐方式】直接配置原生的 logging logger - import logging - jm_logger = logging.getLogger("jmcomic") - - # 例如，取消默认往下游控制台打印的 Handler，转存到文件中 - jm_logger.handlers.clear() - jm_logger.addHandler(logging.FileHandler("jm_download.log", encoding="utf-8")) - - # 【向后兼容方式/遗留用法】 - # 你依然可以通过替换全局配置类的 EXECUTOR_LOG 函数暴力接管日志 - def my_custom_log(topic: str, msg, e: Exception = None): - """ - @param topic: log主题，例如 'album.before', 'req.error', 'plugin.error' - @param msg: 具体log的信息，也可以直接传入 Exception 对象（底层会自动适配） - @param e: 可选，异常对象（当 msg 本身就是 Exception 时无需传） - """ - pass - - # 生效自定义的 log 打印函数 - JmModuleConfig.EXECUTOR_LOG = my_custom_log -``` - - - - -## 自定义异常监听器/回调 - -```python -def custom_exception_listener(): - """ - 该函数演示jmcomic的异常监听器机制 - """ - - # 1. 选一个可能会发生的、你感兴趣的异常 - etype = ResponseUnexpectedException - - - def listener(e): - """ - 你的监听器方法 - 该方法无需返回值 - :param e: 异常实例 - """ - print(f'my exception listener invoke !!! exception happened: {e}') - - - # 注册监听器/回调 - # 这个异常类（或者这个异常的子类）的实例将要被raise前，你的listener方法会被调用 - JmModuleConfig.register_exception_listener(etype, listener) -``` diff --git a/assets/docs/sources/tutorial/6_plugin.md b/assets/docs/sources/tutorial/6_plugin.md index bd8ea92cb..e7034384c 100644 --- a/assets/docs/sources/tutorial/6_plugin.md +++ b/assets/docs/sources/tutorial/6_plugin.md @@ -4,7 +4,7 @@ plugin(扩展/插件)是v2.2.0新引入的机制，使用插件可以实现灵 目前jmcomic已经内置了一些插件，源码位于 src/jmcomic/jm_plugin.py。 -你可以在这里查看这些插件的配置→ [option_file_syntax](../option_file_syntax.md#3-option插件配置项) +你可以在这里查看这些插件的配置→ [option_file_syntax](../option_file_syntax.md#3-option) ## 1. 插件机制介绍 @@ -54,7 +54,59 @@ option = jmcomic.create_option_by_file('xxx.yml') # 创建option对象 option.download_album(123) ``` -## 3. 怎么手动调用插件 +## 3. 综合示例：登录、增量下载与分章压缩 + +下面把多个插件组合起来，完成这些需求： + +1. 使用登录状态下载本子； +2. 只下载指定章节之后的新章节； +3. 将图片转换为 JPG； +4. 每个章节分别压缩，并在压缩成功后删除原图片。 + +先创建并加载配置文件： + +```python +from jmcomic import create_option + +# 加载配置时会执行 after_init 插件，其中 find_update 会开始下载 +create_option('myoption.yml') +``` + +然后在 `myoption.yml` 中组合插件： + +```yaml +dir_rule: + rule: Bd_Aid + base_dir: ./downloads + +download: + image: + suffix: .jpg + +plugins: + after_init: + - plugin: login + kwargs: + username: your_username + password: your_password + + - plugin: find_update + kwargs: + 145504: 290266 # 只下载本子 145504 中章节 290266 之后的新章节 + + after_album: + - plugin: zip + kwargs: + level: photo + filename_rule: Ptitle + zip_dir: ./downloads + delete_original_file: true +``` + +这里的执行顺序是：先登录，再由 `find_update` 发起增量下载；每个本子下载完成后，`zip` 插件按章节生成压缩文件并删除已成功压缩的原图片。 + + +## 4. 怎么手动调用插件 你可以使用下面的代码触发某个事件 @@ -70,7 +122,7 @@ option.call_all_plugin('my_event') ``` -## 4. 示例：自定义插件 +## 5. 示例：自定义插件 * 如果你有好的plugin想法，也欢迎向我提PR，将你的plugin内置到jmcomic模块中 diff --git a/assets/docs/sources/tutorial/7_advance.md b/assets/docs/sources/tutorial/7_advance.md deleted file mode 100644 index 99f640e28..000000000 --- a/assets/docs/sources/tutorial/7_advance.md +++ /dev/null @@ -1,54 +0,0 @@ -# 综合使用实例 - -* 基于v2.2.2+内置插件 - -## 功能需求 - -1. 下载id为145504的本子； -2. 只下载章节在290266之后的新章； -3. 最后要把图片转为jpg格式； -4. 要使用登录状态去下载； -5. 压缩文件，按照章节压缩，一个章节一个压缩文件，压缩文件的命名: 章节标题.zip； -6. 自动把下载的文件全部删除，最后只要保留压缩文件。 - -## 实现方案 - -#### 1. 写2行python代码 -```python -from jmcomic import create_option -create_option('myoption.yml') -``` - -#### 2. 配置option，调用1的脚本即可 -```yaml -dir_rule: # 下载路径规则 - rule: Bd_Aid - base_dir: D:/jmcomic - -download: - image: - suffix: .jpg # 转为jpg格式的图片 - -client: - domain: - - 18comic.vip # 指定域名 - -plugins: - after_init: - - plugin: login # 登录插件 - kwargs: - username: un - password: pw - - - plugin: find_update # 只下载新章插件 - kwargs: - 145504: 290266 # 下载本子145504的章节290266以后的新章 - - after_album: - - plugin: zip # 压缩文件插件 - kwargs: - level: photo # 按照章节，一个章节一个压缩文件 - filename_rule: Ptitle # 压缩文件的命名规则 - zip_dir: D:/jmcomic # 压缩文件存放的文件夹 - delete_original_file: true # 压缩成功后，删除所有原文件和文件夹 -``` diff --git a/assets/docs/sources/tutorial/9_custom_download_dir_name.md b/assets/docs/sources/tutorial/9_custom_download_dir_name.md index 630f8b73e..b73d48796 100644 --- a/assets/docs/sources/tutorial/9_custom_download_dir_name.md +++ b/assets/docs/sources/tutorial/9_custom_download_dir_name.md @@ -17,9 +17,11 @@ plugins: ``` 该示例会把文件夹路径中所有`kyockcho`都变为`きょくちょ`，例如： -`D:/a/[kyockcho]本子名称 - kyockcho/` 改为↓ +```text +旧路径：D:/a/[kyockcho]本子名称 - kyockcho/ +新路径：D:/a/[きょくちょ]本子名称 - きょくちょ/ -`D:/a/[きょくちょ]本子名称 - きょくちょ/` +``` --------------- **_如果上述简单的文本替换无法满足你，或者你需要更灵活的组合逻辑，那么下面的 f-string 语法正适合你。_** @@ -54,7 +56,8 @@ D:/a/b/c/社团学姐/00002.webp - 使用 `/` 分隔（推荐）：`Bd / {Atitle} / {Pname}` - 使用 `_` 分隔（兼容旧写法）：`Bd_{Atitle}_{Pname}` -> [!IMPORTANT] +> 🔖**注意** +> > `/` 和 `_` **二选一**，不可混用。含 `/` 时按 `/` 切分，不含 `/` 时按 `_` 切分。 > > 如果你的文件夹名本身需要包含 `_`，请使用 `/` 作为分隔符，例如：`Bd / {Aid}_{Atitle}` diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index bd3fcdfc7..48eb8ef28 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.7.3' +__version__ = '2.7.4' from .jm_task_context import * from .api import * diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index b1d8ab838..6ef842835 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -1,4 +1,5 @@ import asyncio +from time import perf_counter from .jm_downloader import * from .jm_task_context import bind_jm_task_context, jm_task_context @@ -15,6 +16,12 @@ def _download_type(download_api) -> str: return name +def _finish_download_result(detail, dler, task_started_at): + manifest = dler.manifest_dict[detail] + manifest.duration = perf_counter() - task_started_at + return DownloadResult(detail, dler) + + def download_batch(download_api, jm_id_iter: Union[Iterable, Generator], option=None, @@ -89,15 +96,17 @@ def download_album(jm_album_id, if not isinstance(jm_album_id, (str, int)): return download_batch(download_album, jm_album_id, option, downloader, extra=extra) - with jm_task_context(download_type='album', jm_id=str(jm_album_id)): + task_started_at = perf_counter() + with jm_task_context(download_type='album', jm_id=str(jm_album_id), task_started_at=task_started_at): with new_downloader(option, downloader) as dler: - # 注册 Feature 及来源，由 downloader 在 after_album 钩子中自动执行 - dler.add_features(extra, 'download_album') + # 下载类型已记录在 TaskContext 中，Feature 会据此选择执行钩子 + dler.add_features(extra) album = dler.download_album(jm_album_id) if check_exception: dler.raise_if_has_exception() - return DownloadResult(album, dler) + + return _finish_download_result(album, dler, task_started_at) def download_photo(jm_photo_id, @@ -116,15 +125,17 @@ def download_photo(jm_photo_id, if not isinstance(jm_photo_id, (str, int)): return download_batch(download_photo, jm_photo_id, option, downloader, extra=extra) - with jm_task_context(download_type='photo', jm_id=str(jm_photo_id)): + task_started_at = perf_counter() + with jm_task_context(download_type='photo', jm_id=str(jm_photo_id), task_started_at=task_started_at): with new_downloader(option, downloader) as dler: - # 注册 Feature 及来源，由 downloader 在 after_photo 钩子中自动执行 - dler.add_features(extra, 'download_photo') + # 下载类型已记录在 TaskContext 中，Feature 会据此选择执行钩子 + dler.add_features(extra) photo = dler.download_photo(jm_photo_id) if check_exception: dler.raise_if_has_exception() - return DownloadResult(photo, dler) + + return _finish_download_result(photo, dler, task_started_at) def new_downloader(option=None, downloader=None) -> JmDownloader: @@ -161,12 +172,11 @@ def create_option_by_str(text: str, mode=None): def new_async_downloader(option=None, downloader=None): - from .jm_async_downloader import JmAsyncDownloader if option is None: option = JmModuleConfig.option_class().default() if downloader is None: - downloader = JmAsyncDownloader + downloader = JmModuleConfig.async_downloader_class() return downloader(option) @@ -194,15 +204,16 @@ async def download_album_async(jm_album_id, extra=extra ) - with jm_task_context(download_type='album', jm_id=str(jm_album_id)): + task_started_at = perf_counter() + with jm_task_context(download_type='album', jm_id=str(jm_album_id), task_started_at=task_started_at): async with new_async_downloader(option, downloader) as dler: - dler.add_features(extra, 'download_album') + dler.add_features(extra) album = await dler.download_album(jm_album_id) if check_exception: dler.raise_if_has_exception() - return DownloadResult(album, dler) + return _finish_download_result(album, dler, task_started_at) async def download_photo_async(jm_photo_id, @@ -226,15 +237,16 @@ async def download_photo_async(jm_photo_id, extra=extra ) - with jm_task_context(download_type='photo', jm_id=str(jm_photo_id)): + task_started_at = perf_counter() + with jm_task_context(download_type='photo', jm_id=str(jm_photo_id), task_started_at=task_started_at): async with new_async_downloader(option, downloader) as dler: - dler.add_features(extra, 'download_photo') + dler.add_features(extra) photo = await dler.download_photo(jm_photo_id) if check_exception: dler.raise_if_has_exception() - return DownloadResult(photo, dler) + return _finish_download_result(photo, dler, task_started_at) async def download_batch_async(download_api, diff --git a/src/jmcomic/cl.py b/src/jmcomic/cl.py deleted file mode 100644 index c5dbf3113..000000000 --- a/src/jmcomic/cl.py +++ /dev/null @@ -1,11 +0,0 @@ -import warnings - -from .cli import * - - -warnings.warn( - "The 'jmcomic.cl' module is deprecated and renamed to 'jmcomic.cli'. " - "Please update your imports. It will be removed in version 2.7.4.", - DeprecationWarning, - stacklevel=2, -) diff --git a/src/jmcomic/cli.py b/src/jmcomic/cli.py index 0e9aec6e4..ad1871db8 100644 --- a/src/jmcomic/cli.py +++ b/src/jmcomic/cli.py @@ -12,6 +12,7 @@ $ jmv abc123141 --option="D:/option.yml" """ +import importlib.util import os.path from typing import List, Optional @@ -29,6 +30,7 @@ class JmcomicUI: def __init__(self) -> None: self.option_path: Optional[str] = None + self.progress_enabled = True self.raw_id_list: List[str] = [] self.album_id_list: List[str] = [] self.photo_id_list: List[str] = [] @@ -50,8 +52,15 @@ def parse_arg(self): type=str, default=get_env('JM_OPTION_PATH', ''), ) + parser.add_argument( + '--no-progress', + action='store_false', + dest='progress_enabled', + help='do not automatically enable the download progress plugin', + ) args = parser.parse_args() + self.progress_enabled = args.progress_enabled option = args.option if len(option) == 0 or option == "''": self.option_path = None @@ -82,6 +91,16 @@ def parse(text): def main(self): self.parse_arg() + from .api import create_option, JmOption + from .jm_task_context import jm_task_context + with jm_task_context(cli_no_progress=not self.progress_enabled): + if self.option_path is not None: + option = create_option(self.option_path) + else: + option = JmOption.default() + + self.enable_download_progress(option) + from .api import jm_log jm_log('command_line', f'start downloading...\n' @@ -90,14 +109,30 @@ def main(self): f'- album: {self.album_id_list}\n' f'- photo: {self.photo_id_list}') - from .api import create_option, JmOption - if self.option_path is not None: - option = create_option(self.option_path) - else: - option = JmOption.default() - self.run(option) + def enable_download_progress(self, option): + option_enabled = self.option_has_download_progress(option) + if not self.progress_enabled: + return + + if option_enabled: + return + + if importlib.util.find_spec('rich') is None: + from .jm_config import jm_log + jm_log('command_line.progress', + '未安装 rich，继续使用普通日志。如需显示下载进度，请执行：pip install rich') + return + + from .jm_plugin import DownloadProgressPlugin + DownloadProgressPlugin.build(option).invoke() + + @staticmethod + def option_has_download_progress(option): + plugin_list = option.plugins.get('after_init', []) or [] + return any(plugin.get('plugin') == 'download_progress' for plugin in plugin_list) + def run(self, option): from .api import download_album, download_photo from common import MultiTaskLauncher diff --git a/src/jmcomic/jm_async_client.py b/src/jmcomic/jm_async_client.py index 411f49657..c1dfe834d 100644 --- a/src/jmcomic/jm_async_client.py +++ b/src/jmcomic/jm_async_client.py @@ -7,6 +7,7 @@ import asyncio import json +from copy import deepcopy from typing import Sequence from urllib.parse import urlencode @@ -184,7 +185,6 @@ async def _ensure_session(self): return # 提取应用配置中预设的网络通信元数据信息（如代理配置与全局 Headers） - from copy import deepcopy postman_conf = deepcopy(self.option.client.get('postman', {})) meta_data = postman_conf.get('meta_data', {}) if self._meta_kwargs: @@ -393,7 +393,7 @@ async def _fetch_detail_entity(self, jmid, clazz: type[DetailType]) -> DetailTyp cached = self._cache_get(cache_key) if cached is not self._SENTINEL: # noinspection PyTypeChecker - return cached + return deepcopy(cached) url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER resp = await self.req_api(url, params={'id': jmid}) @@ -402,7 +402,7 @@ async def _fetch_detail_entity(self, jmid, clazz: type[DetailType]) -> DetailTyp ExceptionTool.raise_missing(resp, jmid) result = JmApiAdaptTool.parse_entity(resp.res_data, clazz) - self._cache_set(cache_key, result) + self._cache_set(cache_key, deepcopy(result)) return result async def get_album_detail(self, album_id) -> JmAlbumDetail: @@ -559,9 +559,9 @@ async def search(self, data = resp.model_data if data.get('redirect_aid', None) is not None: aid = data.redirect_aid - result = JmSearchPage.wrap_single_album(await self.get_album_detail(aid)) + result = JmSearchPage.wrap_single_album(await self.get_album_detail(aid), page) else: - result = JmPageTool.parse_api_to_search_page(data) + result = JmPageTool.parse_api_to_search_page(data, page) self._cache_set(cache_key, result) return result @@ -592,7 +592,7 @@ async def categories_filter(self, 'o': o, } resp = await self.req_api(self.API_CATEGORIES_FILTER, params=params) - return JmPageTool.parse_api_to_search_page(resp.model_data) + return JmPageTool.parse_api_to_search_page(resp.model_data, page) # month_ranking / week_ranking / day_ranking # 继承自 AsyncJmcomicClient 基类 @@ -631,7 +631,7 @@ async def favorite_folder(self, 'o': order_by, } ) - return JmPageTool.parse_api_to_favorite_page(resp.model_data) + return JmPageTool.parse_api_to_favorite_page(resp.model_data, page) async def album_pagination(self, jm_id: str, @@ -640,7 +640,7 @@ async def album_pagination(self, with_ad_wcm=1, need_total=True, ) -> JmAlbumCommentPage: - """获取本子评论分页，返回 ``list`` 和 ``total``。""" + """获取本子评论分页，返回评论分页对象。""" resp = await self.req_api( self.API_FORUM, params={ @@ -649,7 +649,21 @@ async def album_pagination(self, 'aid': JmcomicText.parse_to_jm_id(jm_id), }, ) - return JmPageTool.parse_api_to_album_comment_page(resp.model_data) + return JmPageTool.parse_api_to_album_comment_page(resp.model_data, page) + + async def forum_pagination(self, + page=1, + with_ad_wcm=1, + ) -> JmAlbumCommentPage: + """获取全站评论分页。""" + resp = await self.req_api( + self.API_FORUM, + params={ + 'mode': 'all', + 'page': page, + }, + ) + return JmPageTool.parse_api_to_album_comment_page(resp.model_data, page) async def add_favorite_album(self, album_id, folder_id='0'): """ diff --git a/src/jmcomic/jm_async_downloader.py b/src/jmcomic/jm_async_downloader.py index ee0bef025..3f89e24e0 100644 --- a/src/jmcomic/jm_async_downloader.py +++ b/src/jmcomic/jm_async_downloader.py @@ -12,10 +12,10 @@ import os from concurrent.futures import ThreadPoolExecutor -from .jm_downloader import BaseDownloader +from .jm_downloader import BaseDownloader, record_download_duration from .jm_entity import JmAlbumDetail, JmPhotoDetail, JmImageDetail from .jm_toolkit import JmImageTool -from .jm_config import jm_log +from .jm_config import JmModuleConfig, jm_log from .jm_task_context import bind_jm_task_context from .jm_option import JmOption @@ -53,6 +53,18 @@ def __init__(self, # 解密线程池（CPU 密集操作卸载） self._decode_pool = ThreadPoolExecutor(max_workers=decode_worker, thread_name_prefix='jm-async-decode') + @classmethod + def use(cls, *args, **kwargs): + before_class = JmModuleConfig.async_downloader_class() + JmModuleConfig.CLASS_ASYNC_DOWNLOADER = cls + after_class = JmModuleConfig.async_downloader_class() + jm_log( + 'async_downloader.use', + f'替换 Async Downloader class: ' + f'[{before_class.__module__}.{before_class.__qualname__}] -> ' + f'[{after_class.__module__}.{after_class.__qualname__}]' + ) + # ====================================================================== # 核心下载流程 — 对齐 sync JmDownloader # ====================================================================== @@ -65,17 +77,24 @@ async def _run_in_decode_pool(self, func, *args): *args, ) + @record_download_duration('album_started_at') async def download_album(self, album_id) -> JmAlbumDetail: """对齐 sync JmDownloader.download_album""" album = await self.client.get_album_detail(album_id) - await self.download_by_album_detail(album) + self.begin_manifest(album) + try: + await self.download_by_album_detail(album) + finally: + self.finish_manifest(album) return album + @record_download_duration('album_started_at') async def download_by_album_detail(self, album: JmAlbumDetail): """ 异步下载整个本子。 对齐 sync JmDownloader.download_by_album_detail 的回调链路。 """ + album.save_path = self.option.dir_rule.decide_album_root_dir(album) await self.before_album(album) if album.skip: return @@ -99,17 +118,24 @@ async def _safe_download_photo(self, photo: JmPhotoDetail): jm_log('photo.failed', f'章节下载失败: [{photo.id}], 异常: [{e}]', e) self.download_failed_photo.append((photo, e)) + @record_download_duration('photo_started_at') async def download_photo(self, photo_id) -> JmPhotoDetail: """对齐 sync JmDownloader.download_photo""" photo = await self.client.get_photo_detail(photo_id) - await self.download_by_photo_detail(photo) + self.begin_manifest(photo) + try: + await self.download_by_photo_detail(photo) + finally: + self.finish_manifest(photo) return photo + @record_download_duration('photo_started_at') async def download_by_photo_detail(self, photo: JmPhotoDetail): """ 异步下载一个章节的所有图片。 对齐 sync JmDownloader.download_by_photo_detail 的回调链路。 """ + photo.save_path = self.option.decide_image_save_dir(photo) # _photo_semaphore 包裹整段 photo 下载（check_photo + 全部图片）， # 真正限制「同时下载的章节数」（对齐 sync：每个 photo 占用 photo 线程池一个槽位）。 # 章节内图片再由共享的 _image_semaphore 二级限流。 @@ -145,6 +171,7 @@ async def _safe_download_image(self, image: JmImageDetail): jm_log('image.failed', f'图片下载失败: [{image.download_url}], 异常: [{e}]', e) self.download_failed_image.append((image, e)) + @record_download_duration('image_started_at') async def _download_single_image(self, image: JmImageDetail): """ 下载并解密单张图片的完整流程。 @@ -159,8 +186,8 @@ async def _download_single_image(self, image: JmImageDetail): if image.skip: return - # 检查缓存，跳过下载 if image.cache and image.exists: + await self.after_image(image, img_save_path) return decode_image = self.option.decide_download_image_decode(image) diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 1c7b81b45..597148371 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -1,3 +1,4 @@ +from copy import deepcopy from threading import Lock from .jm_task_context import bind_jm_task_context @@ -187,10 +188,10 @@ def cache_wrapper(*args, **kwargs): result = cache.get(key, sentinel) if result is not sentinel: - return result + return deepcopy(result) if isinstance(result, DetailEntity) else result result = func(*args, **kwargs) - cache[key] = result + cache[key] = deepcopy(result) if isinstance(result, DetailEntity) else result return result setattr(self, func_name, cache_wrapper) @@ -243,6 +244,7 @@ class JmHtmlClient(AbstractJmClient): API_SEARCH = '/search/photos' API_CATEGORY = '/albums' API_ALBUM_PAGINATION = '/ajax/album_pagination' + API_FORUM = '/ajax/forum_more' def add_favorite_album(self, album_id, @@ -335,9 +337,9 @@ def search(self, # 因为如果搜索的是禁漫车号，会直接跳转到本子详情页面 if resp.redirect_count != 0 and '/album/' in resp.url: album = JmcomicText.analyse_jm_album_html(resp.text) - return JmSearchPage.wrap_single_album(album) + return JmSearchPage.wrap_single_album(album, page) else: - return JmPageTool.parse_html_to_search_page(resp.text) + return JmPageTool.parse_html_to_search_page(resp.text, page) @classmethod def build_search_url(cls, base: str, category: str, sub_category: Optional[str]): @@ -378,7 +380,7 @@ def categories_filter(self, allow_redirects=True, ) - return JmPageTool.parse_html_to_category_page(resp.text) + return JmPageTool.parse_html_to_category_page(resp.text, page) # -- 帐号管理 -- @@ -438,7 +440,7 @@ def favorite_folder(self, } ) - return JmPageTool.parse_html_to_favorite_page(resp.text) + return JmPageTool.parse_html_to_favorite_page(resp.text, page) # noinspection PyTypeChecker def get_username_from_cookies(self) -> str: @@ -553,13 +555,28 @@ def album_pagination(self, ret = JmJsonResp(resp) ret.require_success() - comment_page = JmPageTool.parse_html_to_album_comment_page(ret.model()) + comment_page = JmPageTool.parse_html_to_album_comment_page(ret.model(), page) if need_total: album = self.get_album_detail(jm_id) comment_page.total = album.comment_count return comment_page + def forum_pagination(self, + page=1, + with_ad_wcm=1, + ) -> JmAlbumCommentPage: + resp = self.post( + self.API_FORUM, + data={ + 'page': page, + 'with_ad_wcm': with_ad_wcm, + }, + ) + ret = JmJsonResp(resp) + ret.require_success() + return JmPageTool.parse_html_to_album_comment_page(ret.model(), page) + @classmethod def require_resp_success_else_raise(cls, resp, url: str): """ @@ -672,9 +689,9 @@ def search(self, data = resp.model_data if data.get('redirect_aid', None) is not None: aid = data.redirect_aid - return JmSearchPage.wrap_single_album(self.get_album_detail(aid)) + return JmSearchPage.wrap_single_album(self.get_album_detail(aid), page) - return JmPageTool.parse_api_to_search_page(data) + return JmPageTool.parse_api_to_search_page(data, page) def categories_filter(self, page: int, @@ -698,7 +715,7 @@ def categories_filter(self, resp = self.req_api(self.append_params_to_url(self.API_CATEGORIES_FILTER, params)) - return JmPageTool.parse_api_to_search_page(resp.model_data) + return JmPageTool.parse_api_to_search_page(resp.model_data, page) def get_album_detail(self, album_id) -> JmAlbumDetail: return self.fetch_detail_entity(album_id, @@ -883,7 +900,7 @@ def favorite_folder(self, } ) - return JmPageTool.parse_api_to_favorite_page(resp.model_data) + return JmPageTool.parse_api_to_favorite_page(resp.model_data, page) def album_comment(self, video_id, @@ -910,7 +927,20 @@ def album_pagination(self, 'aid': JmcomicText.parse_to_jm_id(jm_id), }, ) - return JmPageTool.parse_api_to_album_comment_page(resp.model_data) + return JmPageTool.parse_api_to_album_comment_page(resp.model_data, page) + + def forum_pagination(self, + page=1, + with_ad_wcm=1, + ) -> JmAlbumCommentPage: + resp = self.req_api( + self.API_FORUM, + params={ + 'mode': 'all', + 'page': page, + }, + ) + return JmPageTool.parse_api_to_album_comment_page(resp.model_data, page) def add_favorite_album(self, album_id, diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index 01cab739f..0c08347a0 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -232,7 +232,7 @@ def album_pagination(self, """ 获取本子评论分页。 - 推荐直接遍历评论页，并通过 ``comment.replies`` 读取回评： + 推荐直接遍历评论页，并通过 comment.replies 读取回评： ``` def show_comment(comment, indent=''): @@ -254,13 +254,13 @@ def show_comment(comment, indent=''): 返回字段说明： - - ``page.total``：全部分页的主评论总数。 - - ``page.comment_count``：当前页主评论加所有层级回评的数量。 - - ``len(page)``：当前页主评论数量，不包含回评。 - - ``comment.raw_data``、``page.raw_data``：统一为 ``AdvancedDict``。 - - HTML 客户端的 ``comment.likes`` 固定为 ``None``。 - - HTML 客户端的 ``comment.user_id`` 仅在头像地址包含数字 ID 时可用。 - - HTML 客户端的 ``comment.album_id`` 在评论 DOM 缺少本子链接时为 ``None``。 + - page.total：全部分页的主评论总数。 + - page.comment_count：当前页主评论加所有层级回评的数量。 + - len(page)：当前页主评论数量，不包含回评。 + - comment.raw_data、page.raw_data：统一为 AdvancedDict。 + - HTML 客户端的 comment.likes 固定为 None。 + - HTML 客户端的 comment.user_id 仅在头像地址包含数字 ID 时可用。 + - HTML 客户端的 comment.album_id 在评论 DOM 缺少本子链接时为 None。 :param jm_id: album_id/photo_id :param page: 页码，从 1 开始 @@ -271,6 +271,23 @@ def show_comment(comment, indent=''): """ raise NotImplementedError + def forum_pagination(self, + page=1, + with_ad_wcm=1, + ) -> JmAlbumCommentPage: + """ + 获取全站评论分页。 + + HTML 客户端使用 /ajax/forum_more；API 客户端使用不带 aid 的 /forum。 + HTML 客户端不提供评论总数，page.total 和 page.page_count 为 None； + API 客户端返回全站主评论总数。 + + :param page: 页码，从 1 开始 + :param with_ad_wcm: 网页端广告标记，API 端忽略 + :return: JmAlbumCommentPage + """ + raise NotImplementedError + def favorite_folder(self, page=1, order_by=JmMagicConstants.ORDER_BY_LATEST, @@ -645,12 +662,47 @@ def album_pagination_gen(self, if comment_page.page_count is not None: if page >= comment_page.page_count: break - elif not any( - int(match.group(1)) == page + 1 - for match in JmcomicText.pattern_html_comment_next_page.finditer(comment_page.raw_html or '') - ): + else: + # HTML 响应没有评论总数，通过分页按钮中是否存在下一页判断是否继续。 + has_next_page = any( + int(match.group(1)) == page + 1 + for match in JmcomicText.pattern_html_comment_next_page.finditer(comment_page.raw_html or '') + ) + if not has_next_page: + break + + page += 1 + + def forum_pagination_gen(self, + page=1, + with_ad_wcm=1, + ) -> Generator[JmAlbumCommentPage, None, None]: + """逐页获取全站评论；HTML 客户端遇到服务端重复返回的末页时停止。""" + # 仅供 HTML 客户端使用；API 客户端通过 total 计算 page_count。 + previous_comment_ids = None + while True: + comment_page = self.forum_pagination( + page=page, + with_ad_wcm=with_ad_wcm, + ) + comment_ids = tuple(comment.comment_id for comment in comment_page) + if not comment_ids: + break + + # HTML 响应没有 total/has_more，只能通过重复页识别末页。 + if comment_page.page_count is None and comment_ids == previous_comment_ids: break + yield comment_page + + if comment_page.page_count is not None: + # API 响应包含 total，page_count 可直接判断末页。 + if page >= comment_page.page_count: + break + else: + # HTML 客户端保存本页 ID，供下一次请求检测重复页。 + previous_comment_ids = comment_ids + page += 1 def search_gen(self, @@ -1018,7 +1070,18 @@ async def album_pagination(self, 获取本子评论分页。 异步客户端使用移动端 API，返回值与同步 JmApiClient 一致； - ``series`` 和 ``with_ad_wcm`` 仅用于保持同步/异步签名一致。 + series 和 with_ad_wcm 仅用于保持同步、异步方法签名一致。 + """ + raise NotImplementedError + + async def forum_pagination(self, + page=1, + with_ad_wcm=1, + ) -> JmAlbumCommentPage: + """ + 异步获取全站评论分页。 + + with_ad_wcm 仅用于保持同步、异步方法签名一致。 """ raise NotImplementedError @@ -1057,6 +1120,33 @@ async def album_pagination_gen(self, page += 1 + async def forum_pagination_gen(self, + page=1, + with_ad_wcm=1, + ): + """异步逐页获取全站评论；异步客户端当前只有 API 实现。""" + # API 正常通过 total 计算 page_count；保存上一页 ID 仅用于异常响应缺少 total 时兜底。 + previous_comment_ids = None + while True: + comment_page = await self.forum_pagination( + page=page, + with_ad_wcm=with_ad_wcm, + ) + comment_ids = tuple(comment.comment_id for comment in comment_page) + if not comment_ids: + break + + if comment_page.page_count is None and comment_ids == previous_comment_ids: + break + + yield comment_page + + if comment_page.page_count is not None and page >= comment_page.page_count: + break + + previous_comment_ids = comment_ids + page += 1 + # -- 域名 / 缓存管理 -- def get_domain_list(self) -> List[str]: diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index a1ad3ed50..4dd5442b8 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -253,6 +253,7 @@ class JmModuleConfig: # 模块级别的可重写类配置 CLASS_DOWNLOADER = None + CLASS_ASYNC_DOWNLOADER = None CLASS_OPTION = None CLASS_ALBUM = None CLASS_PHOTO = None @@ -309,6 +310,14 @@ def downloader_class(cls): from .jm_downloader import JmDownloader return JmDownloader + @classmethod + def async_downloader_class(cls): + if cls.CLASS_ASYNC_DOWNLOADER is not None: + return cls.CLASS_ASYNC_DOWNLOADER + + from .jm_async_downloader import JmAsyncDownloader + return JmAsyncDownloader + @classmethod def option_class(cls): if cls.CLASS_OPTION is not None: diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index f760d5724..2846e547b 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -1,12 +1,81 @@ +import os +import inspect +from functools import wraps from typing import NamedTuple +from time import perf_counter from .jm_option import * -from .jm_task_context import bind_jm_task_context +from .jm_task_context import bind_jm_task_context, get_jm_task_context, jm_task_context + + +def record_download_duration(context_key: str, clock=None): + def decorator(func): + # 装饰时只解析一次参数名，关键字调用无需在每次执行时重复 inspect。 + entity_param = tuple(inspect.signature(func).parameters)[1] + + def get_time(): + return perf_counter() if clock is None else clock() + + def get_entity(args, kwargs): + # 常规位置参数走快路径；只有关键字调用才按参数名取值。 + if len(args) > 1: + return args[1] + return kwargs[entity_param] + + if inspect.iscoroutinefunction(func): + @wraps(func) + async def async_wrapper(*args, **kwargs): + entity = get_entity(args, kwargs) + detail_call = isinstance(entity, Downloadable) + # 顶层 ID 下载负责完整耗时，内部 detail 调用复用同一个计时上下文。 + if detail_call and get_jm_task_context().get(context_key) is not None: + return await func(*args, **kwargs) + + started_at = get_time() + with jm_task_context(**{context_key: started_at}): + result = await func(*args, **kwargs) + # detail 入口直接记录传入实体；ID 入口记录下载后返回的实体。 + detail = entity if detail_call else result + detail.duration = get_time() - started_at + return result + + return async_wrapper + + @wraps(func) + def wrapper(*args, **kwargs): + entity = get_entity(args, kwargs) + detail_call = isinstance(entity, Downloadable) + # 顶层 ID 下载负责完整耗时，内部 detail 调用复用同一个计时上下文。 + if detail_call and get_jm_task_context().get(context_key) is not None: + return func(*args, **kwargs) + + started_at = get_time() + with jm_task_context(**{context_key: started_at}): + result = func(*args, **kwargs) + # detail 入口直接记录传入实体；ID 入口记录下载后返回的实体。 + detail = entity if detail_call else result + detail.duration = get_time() - started_at + return result + + return wrapper + + return decorator + + +class DownloadManifest: + """一次顶层下载产生的聚合文件清单。""" + def __init__(self): + self.image_filepath_list: List[str] = [] + self.export_filepath_dict: Dict[str, List[str]] = {} + self.duration: Optional[float] = None # 顶层任务完整耗时（秒） + + def get_export_filepath_list(self, suffix: str) -> List[str]: + normalized_suffix = str(suffix).lower().lstrip('.') + return self.export_filepath_dict.get(normalized_suffix, []) -def catch_exception(func): - from functools import wraps +def catch_exception(func): @wraps(func) def wrapper(self, *args, **kwargs): self: JmDownloader @@ -84,8 +153,10 @@ def __init__(self, option: JmOption): # 下载失败的记录list self.download_failed_image: List[Tuple[JmImageDetail, BaseException]] = [] self.download_failed_photo: List[Tuple[JmPhotoDetail, BaseException]] = [] - # Feature 特性列表: [(feature, feature_from), ...] - self._feature_list: List[Tuple] = [] + # 每次顶层下载对应的聚合清单 + self.manifest_dict: Dict[DetailEntity, DownloadManifest] = {} + # 当前顶层下载注册的 Feature 列表 + self._feature_list: List = [] def do_filter(self, detail: DetailEntity): """ @@ -176,37 +247,99 @@ def before_image(self, image: JmImageDetail, img_save_path): def after_image(self, image: JmImageDetail, img_save_path): super().after_image(image, img_save_path) - photo = image.from_photo - album = photo.from_album - - self.download_success_dict.get(album).get(photo).append((img_save_path, image)) self.option.call_all_plugin( 'after_image', image=image, downloader=self, ) + photo = image.from_photo + album = photo.from_album + self.download_success_dict.get(album).get(photo).append((image.save_path, image)) + + def begin_manifest(self, detail: DetailEntity) -> DownloadManifest: + manifest = DownloadManifest() + self.manifest_dict[detail] = manifest + return manifest + + def resolve_manifest_detail(self, detail: DetailEntity) -> Optional[DetailEntity]: + if detail in self.manifest_dict: + return detail + + if detail.is_photo() and detail.from_album in self.manifest_dict: + return detail.from_album + + return None + + def record_export_filepath(self, detail: DetailEntity, filepath: str) -> None: + manifest_detail = self.resolve_manifest_detail(detail) + if manifest_detail is None: + from .jm_toolkit import ExceptionTool + ExceptionTool.raises(f'当前实体没有活动的下载清单: {detail}') + + suffix = os.path.splitext(filepath)[1].lower().lstrip('.') + if suffix == '': + return + + manifest = self.manifest_dict[manifest_detail] + manifest.export_filepath_dict.setdefault(suffix, []).append(filepath) + + def finish_manifest(self, detail: DetailEntity) -> DownloadManifest: + manifest = self.manifest_dict[detail] + if detail.is_album(): + success_dict = self.download_success_dict.get(detail, {}) + success_groups = [ + success_list + for _, success_list in sorted( + success_dict.items(), + key=lambda item: item[0].index, + ) + ] + else: + success_groups = [ + self.download_success_dict.get(detail.from_album, {}).get(detail, []) + ] + + manifest.image_filepath_list = [ + image.save_path + for success_list in success_groups + for _, image in sorted(success_list, key=lambda item: item[1].index) + ] + return manifest + + @staticmethod + def _require_feature_context() -> str: + from .jm_toolkit import ExceptionTool + + download_type = get_jm_task_context().get('download_type') + ExceptionTool.require_true( + download_type in ('album', 'photo'), + 'Feature 注册与执行必须位于下载任务上下文中，请使用 ' + "jm_task_context(download_type='album') 或 " + "jm_task_context(download_type='photo') 进行包裹", + ) + return download_type - def add_features(self, features, feature_from: str): + def add_features(self, features): """ - 注册 Feature 及其来源。 + 为当前顶层下载注册 Feature。 :param features: Feature / FeatureChain / list / None - :param feature_from: 来源标记，如 'download_album' 或 'download_photo' """ if features is None: return from .jm_feature import FeatureChain, Feature from .jm_toolkit import ExceptionTool + self._require_feature_context() if isinstance(features, list): for f in features: - self.add_features(f, feature_from) + self.add_features(f) elif isinstance(features, FeatureChain): for f in features.to_list(): - self._feature_list.append((f, feature_from)) + self._feature_list.append(f) elif isinstance(features, Feature): - self._feature_list.append((features, feature_from)) + self._feature_list.append(features) else: ExceptionTool.raises(f'不支持的 extra 类型: {type(features)}，请传入 Feature / FeatureChain / list / None') @@ -217,12 +350,16 @@ def _invoke_features_for(self, when: str, **kwargs): :param when: 当前钩子名，如 'after_album', 'after_photo' :param kwargs: album, photo, downloader 等上下文 """ - for feature, feature_from in self._feature_list: - if feature.should_invoke(feature_from, when): + if len(self._feature_list) == 0: + return + + download_type = self._require_feature_context() + for feature in self._feature_list: + if feature.should_invoke(when): try: - feature.invoke(self.option, feature_from=feature_from, when=when, **kwargs) + feature.invoke(self.option, when=when, **kwargs) except Exception as e: - jm_log('downloader.feature.exception', f'Feature执行失败: [{feature}], 来源: [{feature_from}], 异常: [{e}]', + jm_log('downloader.feature.exception', f'Feature执行失败: [{feature}], 下载类型: [{download_type}], 异常: [{e}]', e) def raise_if_has_exception(self): @@ -253,6 +390,15 @@ class DownloadResult(NamedTuple): detail: DetailEntity downloader: BaseDownloader + @property + def manifest(self) -> DownloadManifest: + return self.downloader.manifest_dict[self.detail] + + @property + def duration(self) -> Optional[float]: + """顶层下载耗时，单位：秒。""" + return self.manifest.duration + class BatchResult(set): """批量下载结果集。 @@ -290,12 +436,19 @@ def create_client(self): """ return self.option.build_jm_client() + @record_download_duration('album_started_at') def download_album(self, album_id): album = self.client.get_album_detail(album_id) - self.download_by_album_detail(album) + self.begin_manifest(album) + try: + self.download_by_album_detail(album) + finally: + self.finish_manifest(album) return album + @record_download_duration('album_started_at') def download_by_album_detail(self, album: JmAlbumDetail): + album.save_path = self.option.dir_rule.decide_album_root_dir(album) self.before_album(album) if album.skip: return @@ -306,15 +459,21 @@ def download_by_album_detail(self, album: JmAlbumDetail): ) self.after_album(album) + @record_download_duration('photo_started_at') def download_photo(self, photo_id): photo = self.client.get_photo_detail(photo_id) - self.download_by_photo_detail(photo) + self.begin_manifest(photo) + try: + self.download_by_photo_detail(photo) + finally: + self.finish_manifest(photo) return photo @catch_exception + @record_download_duration('photo_started_at') def download_by_photo_detail(self, photo: JmPhotoDetail): + photo.save_path = self.option.decide_image_save_dir(photo) self.client.check_photo(photo) - self.before_photo(photo) if photo.skip: return @@ -326,25 +485,22 @@ def download_by_photo_detail(self, photo: JmPhotoDetail): self.after_photo(photo) @catch_exception + @record_download_duration('image_started_at') def download_by_image_detail(self, image: JmImageDetail): img_save_path = self.option.decide_image_filepath(image) - image.save_path = img_save_path image.exists = file_exists(img_save_path) image.cache = self.option.decide_download_cache(image) self.before_image(image, img_save_path) - if image.skip: return - # let option decide use_cache and decode_image - decode_image = self.option.decide_download_image_decode(image) - - # skip download if image.cache and image.exists: + self.after_image(image, img_save_path) return + decode_image = self.option.decide_download_image_decode(image) self.client.download_by_image_detail( image, img_save_path, @@ -399,7 +555,15 @@ def use(cls, *args, **kwargs): """ 让本类替换JmModuleConfig.CLASS_DOWNLOADER """ + before_class = JmModuleConfig.downloader_class() JmModuleConfig.CLASS_DOWNLOADER = cls + after_class = JmModuleConfig.downloader_class() + jm_log( + 'downloader.use', + f'替换 Downloader class: ' + f'[{before_class.__module__}.{before_class.__qualname__}] -> ' + f'[{after_class.__module__}.{after_class.__qualname__}]' + ) class DoNotDownloadImage(JmDownloader): diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index 7299807b0..15dfc4bf7 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -9,10 +9,11 @@ class Downloadable: def __init__(self): - self.save_path: str = '' - self.exists: bool = False - self.skip = False - self.cache = True + self.save_path: str = '' # 下载保存路径 + self.exists: bool = False # 下载前，目标是否已存在 + self.skip = False # 是否跳过本次下载，可供外界控制 + self.cache = True # 下载前目标已存在时，是否使用缓存，如果 exists and cache 都是 True，会跳过下载 + self.duration: Optional[float] = None # 下载耗时，单位：秒 class JmBaseEntity: @@ -581,7 +582,7 @@ def is_album(cls): class JmPageContent(JmBaseEntity, IndexedEntity): ContentItem = Tuple[str, Dict[str, Any]] - def __init__(self, content: List[ContentItem], total: int): + def __init__(self, content: List[ContentItem], total: int, page_number: Optional[int] = None): """ content: @@ -590,9 +591,11 @@ def __init__(self, content: List[ContentItem], total: int): ] :param content: 分页数据 :param total: 总结果数 + :param page_number: 当前页码 """ self.content = content self.total = total + self.page_number = page_number # 当前页码，独立解析分页内容时可能为空 @property def page_count(self) -> int: @@ -655,12 +658,12 @@ class JmAlbumComment(JmBaseEntity): """ 本子评论实体。 - ``raw_data`` 始终为 ``AdvancedDict``。 + raw_data 始终为 AdvancedDict。 网页端字段说明： - - ``likes``：评论分页 HTML 不提供，值为 ``None``。 - - ``user_id``：仅头像地址包含数字用户 ID 时可解析，否则为 ``None``。 - - ``album_id``：评论 DOM 缺少本子链接时为 ``None``。 + - likes：评论分页 HTML 不提供，值为 None。 + - user_id：优先读取评论节点的用户 ID，否则尝试从头像地址解析。 + - album_id：从评论中的本子或章节链接解析，链接缺失时为 None。 """ def __init__(self, raw_data): @@ -697,14 +700,21 @@ def __init__(self, raw_data): for reply in data.get('replys', []) or [] ] + def __str__(self): + author = self.nickname or self.username or '' + spoiler = '（剧透）' if self.is_spoiler else '' + return f'[{self.comment_id}] {author}{spoiler}: {self.content or ""}' + + __repr__ = __str__ + class JmAlbumCommentPage(JmBaseEntity, IndexedEntity): """ 本子评论分页。 - ``total`` 是全部分页的主评论总数；``comment_count`` 是当前页主评论 - 加所有层级回评的数量；``len(page)`` 只统计当前页主评论。 - ``raw_data`` 始终为 ``AdvancedDict``。 + total 是全部分页的主评论总数；comment_count 是当前页主评论 + 加所有层级回评的数量；len(page) 只统计当前页主评论。 + page_number 是当前页码；raw_data 始终为 AdvancedDict。 """ def __init__(self, @@ -712,9 +722,12 @@ def __init__(self, total: Optional[int] = None, raw_html: Optional[str] = None, raw_data=None, + *, + page_number: Optional[int] = None, ): self.content = content self.total = total + self.page_number = page_number self.raw_html = raw_html self.raw_data = AdvancedDict.wrap(raw_data or {}) @@ -764,13 +777,13 @@ def single_album(self) -> JmAlbumDetail: return getattr(self, 'album') @classmethod - def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage': + def wrap_single_album(cls, album: JmAlbumDetail, page_number: Optional[int] = None) -> 'JmSearchPage': page = JmSearchPage([( album.album_id, { 'name': album.name, 'tags': album.tags, } - )], 1) + )], 1, page_number) setattr(page, 'album', album) return page @@ -780,14 +793,15 @@ def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage': class JmFavoritePage(JmPageContent): - def __init__(self, content, folder_list, total): + def __init__(self, content, folder_list, total, page_number: Optional[int] = None): """ :param content: 收藏夹一页数据 :param folder_list: 所有的收藏夹的信息 :param total: 收藏夹的收藏总数 + :param page_number: 当前页码 """ - super().__init__(content, total) + super().__init__(content, total, page_number) self.folder_list = folder_list @property diff --git a/src/jmcomic/jm_feature.py b/src/jmcomic/jm_feature.py index 16d464efa..d93bf7dac 100644 --- a/src/jmcomic/jm_feature.py +++ b/src/jmcomic/jm_feature.py @@ -26,7 +26,7 @@ class Feature: 下载完成后自动执行。 Feature 记录在 downloader 上，由 downloader 在 after_album / after_photo - 钩子中根据 feature_from 自动判断是否执行。 + 钩子中根据当前 TaskContext 的 download_type 自动判断是否执行。 """ # 类型声明（保证 IDE 自动补全） @@ -39,23 +39,21 @@ def jm_task_context(self) -> dict: """Return the current invocation's isolated task-context snapshot.""" return get_jm_task_context() - def should_invoke(self, feature_from: str, when: str) -> bool: + def should_invoke(self, when: str) -> bool: """ - 判断在当前钩子(when)下，根据来源(feature_from)，是否应该执行。 + 判断在当前钩子(when)下是否应该执行。 默认返回 True（任何钩子都执行）。子类可覆写来限制执行时机。 - :param feature_from: Feature 的注册来源，如 'download_album', 'download_photo' :param when: 当前触发的钩子名称，如 'after_album', 'after_photo' :returns: 是否应该执行 """ return True - def invoke(self, option: JmOption, feature_from: str, when: str, **kwargs): + def invoke(self, option: JmOption, when: str, **kwargs): """ 执行此 Feature。子类需实现该方法。 :param option: 当前的 JmOption - :param feature_from 注册来源，如 'download_album', 'download_photo' :param when: 钩子回调时机，如 'after_album', 'after_photo' :param kwargs: album, photo, downloader 等回调参数 """ @@ -79,23 +77,20 @@ def to_list(self): class PluginFeature(Feature): """ 插件特性。封装 jmcomic 的插件，在 invoke 时调用相应的插件类。 - 参数根据 feature_from 动态适配，无需写死。 + 参数根据当前钩子动态适配，无需写死。 """ def __init__(self, plugin_key, **kwargs): self.plugin_key = plugin_key self.kwargs = dict(kwargs) - def should_invoke(self, feature_from: str, when: str) -> bool: + def should_invoke(self, when: str) -> bool: """ - 默认根据注册来源推导执行时机： - download_album → after_album, download_photo → after_photo + 默认根据 TaskContext 中的顶层下载类型推导执行时机： + album → after_album, photo → after_photo """ - if feature_from == 'download_album': - return when == 'after_album' - elif feature_from == 'download_photo': - return when == 'after_photo' - return False + download_type = self.jm_task_context.get('download_type') + return when == f'after_{download_type}' def __call__(self, **kwargs): """带自定义参数，返回新实例（继承默认参数）""" @@ -104,16 +99,15 @@ def __call__(self, **kwargs): new_instance = type(self)(self.plugin_key, **new_kwargs) return new_instance - def invoke(self, option: JmOption, feature_from: str, when: str, **extra): + def invoke(self, option: JmOption, when: str, **extra): """ 执行此 Feature 对应的插件。 - 根据 feature_from 动态适配 filename_rule 等参数。 + 根据当前钩子动态适配 filename_rule 等参数。 """ pclass: type = JmModuleConfig.REGISTRY_PLUGIN.get(self.plugin_key) - ExceptionTool.require_true(pclass is not None, f'PluginFeature 引用了未注册的插件: {self.plugin_key}, from {feature_from}, when {when}') + ExceptionTool.require_true(pclass is not None, f'PluginFeature 引用了未注册的插件: {self.plugin_key}, when {when}') - # 根据 feature_from 动态适配参数 - plugin_kwargs: dict = self._adapt_plugin_kwargs(option, feature_from, when) + plugin_kwargs: dict = self._adapt_plugin_kwargs(option, when) option.invoke_plugin( pclass=pclass, @@ -122,9 +116,9 @@ def invoke(self, option: JmOption, feature_from: str, when: str, **extra): pinfo={'plugin': self.plugin_key, 'kwargs': plugin_kwargs}, ) - def _adapt_plugin_kwargs(self, option: JmOption, feature_from: str, when: str) -> dict: + def _adapt_plugin_kwargs(self, option: JmOption, when: str) -> dict: """ - 根据feature_from和when动态确定以下插件参数: + 根据 when 动态确定以下插件参数: filename_rule """ kwargs = self.kwargs.copy() diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 93721e8ca..913e9a956 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -2,7 +2,12 @@ 该文件存放的是option插件 """ +from collections import deque +from threading import RLock + from .jm_option import * +from .jm_async_downloader import JmAsyncDownloader +from .jm_downloader import JmDownloader from .jm_task_context import bind_jm_task_context, get_jm_task_context @@ -353,11 +358,13 @@ def invoke(self, if level == 'album': zip_path = self.decide_filepath(album, None, filename_rule, suffix, zip_dir, dir_rule) self.zip_album(album, photo_dict, zip_path, path_to_delete, encrypt) + downloader.record_export_filepath(album, zip_path) elif level == 'photo': for photo, image_list in photo_dict.items(): zip_path = self.decide_filepath(photo.from_album, photo, filename_rule, suffix, zip_dir, dir_rule) self.zip_photo(photo, image_list, zip_path, path_to_delete, encrypt) + downloader.record_export_filepath(photo, zip_path) else: ExceptionTool.raises(f'Not Implemented Zip Level: {level}') @@ -588,6 +595,427 @@ def invoke(self, whitelist) -> None: jm_logger.addFilter(LogTopicFilterPlugin.TopicFilter(whitelist)) +# noinspection attribute-outside-init +class ProgressDownloader(JmDownloader): + progress_console = None + progress_log_lines = deque(maxlen=6) + active_progresses = set() + progress_ui_lock = RLock() + + @staticmethod + def display_id(entity_id): + entity_id = str(entity_id) + return entity_id if entity_id.upper().startswith('JM') else f'JM{entity_id}' + + @classmethod + def get_progress_console(cls): + if cls.progress_console is None: + from rich.console import Console + cls.progress_console = Console() + return cls.progress_console + + @classmethod + def reset_progress_logs(cls): + with cls.progress_ui_lock: + cls.progress_log_lines.clear() + + @classmethod + def append_progress_log(cls, message): + with cls.progress_ui_lock: + cls.progress_log_lines.append(message) + progresses = tuple(cls.active_progresses) + + for progress in progresses: + try: + progress.refresh() + except Exception: + pass + + @classmethod + def register_progress(cls, progress): + if progress.console.is_interactive: + with cls.progress_ui_lock: + cls.active_progresses.add(progress) + + @classmethod + def unregister_progress(cls, progress): + with cls.progress_ui_lock: + cls.active_progresses.discard(progress) + + @classmethod + def build_log_panel(cls): + from rich.console import Group + from rich.panel import Panel + from rich.text import Text + + with cls.progress_ui_lock: + lines = list(cls.progress_log_lines) + + render_lines = [ + Text(line, overflow='ellipsis', no_wrap=True) + for line in lines + ] + render_lines.extend(Text('') for _ in range(6 - len(render_lines))) + return Panel( + Group(*render_lines), + title='[bold cyan]JMComic Logs[/bold cyan]', + border_style='cyan', + height=8, + ) + + @classmethod + def new_rich_progress(cls, console): + from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + ) + + class ProgressWithLogs(Progress): + def get_renderables(self): + yield cls.build_log_panel() + yield from super().get_renderables() + + return ProgressWithLogs( + SpinnerColumn(style='bright_cyan'), + TextColumn('{task.description}'), + BarColumn( + bar_width=28, + style='grey37', + complete_style='bright_cyan', + finished_style='bright_green', + ), + MofNCompleteColumn(), + TaskProgressColumn(), + TimeElapsedColumn(), + console=console, + auto_refresh=False, + disable=not console.is_interactive, + ) + + def refresh_progress(self): + if self.progress.console.is_interactive: + self.progress.refresh() + + def print_non_interactive_summary(self, photo_id=None): + if self.progress.console.is_interactive: + return + + if photo_id is not None: + done = self.chapter_done[photo_id] + total = self.chapter_total[photo_id] + succeeded = done == total + icon = '✓' if succeeded else '⚠' + status = '下载完成' if succeeded else '下载结束' + chapter_name = f'章节-{ProgressDownloader.display_id(photo_id)}' + self.progress.console.print( + f'{icon} {status}：{chapter_name}，图片 {done}/{total}' + ) + return + + image_done = sum(self.chapter_done.values()) + image_total = sum(self.chapter_total.values()) + succeeded = self.album_done == self.album_total and image_done == image_total + icon = '✓' if succeeded else '⚠' + status = '下载完成' if succeeded else '下载结束' + album_name = f'本子-{ProgressDownloader.display_id(self.album_id)}' + self.progress.console.print( + f'{icon} {status}：{album_name}，章节 {self.album_done}/{self.album_total}，' + f'图片 {image_done}/{image_total}' + ) + + def before_album(self, album): + super().before_album(album) + self.start_progress(len(album), album.id) + + def start_progress(self, album_total=None, album_id=None): + from threading import Lock + + self.progress_lock = Lock() + self.album_total = album_total + self.album_id = album_id + self.album_done = 0 + self.chapter_done = {} + self.chapter_total = {} + self.chapter_tasks = {} + console = self.get_progress_console() + self.progress = self.new_rich_progress(console) + self.progress.start() + self.register_progress(self.progress) + self.album_task = None + if album_total is not None: + self.album_task = self.progress.add_task( + f'[bold magenta]本子-{self.display_id(album_id)}[/bold magenta]', + total=album_total, + ) + self.refresh_progress() + + def before_photo(self, photo): + super().before_photo(photo) + if getattr(self, 'progress', None) is None: + self.start_progress() + with self.progress_lock: + self.chapter_done[photo.id] = 0 + self.chapter_total[photo.id] = len(photo) + self.chapter_tasks[photo.id] = self.progress.add_task( + f'[cyan]章节-{self.display_id(photo.id)}[/cyan]', + total=len(photo), + ) + self.refresh_progress() + + def after_image(self, image, img_save_path): + super().after_image(image, img_save_path) + photo_id = image.from_photo.id + with self.progress_lock: + self.chapter_done[photo_id] += 1 + self.progress.advance(self.chapter_tasks[photo_id]) + self.refresh_progress() + + def after_photo(self, photo): + super().after_photo(photo) + with self.progress_lock: + done = self.chapter_done[photo.id] + succeeded = done == len(photo) + color = 'bold green' if succeeded else 'bold yellow' + icon = '✓' if succeeded else '⚠' + self.progress.update( + self.chapter_tasks[photo.id], + description=f'[{color}]{icon} 章节-{self.display_id(photo.id)}[/{color}]', + ) + if succeeded: + self.album_done += 1 + if self.album_task is not None: + self.progress.advance(self.album_task) + self.refresh_progress() + if self.album_total is None: + self.print_non_interactive_summary(photo.id) + + def after_album(self, album): + super().after_album(album) + with self.progress_lock: + succeeded = self.album_done == len(album) + color = 'bold green' if succeeded else 'bold yellow' + icon = '✓' if succeeded else '⚠' + self.progress.update( + self.album_task, + description=f'[{color}]{icon} 本子-{self.display_id(self.album_id)}[/{color}]', + ) + self.refresh_progress() + self.print_non_interactive_summary() + self.stop_progress() + + def stop_progress(self): + progress = getattr(self, 'progress', None) + if progress is None: + return + self.unregister_progress(progress) + if progress.console.is_interactive: + progress.stop() + self.progress = None + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop_progress() + return super().__exit__(exc_type, exc_val, exc_tb) + +# noinspection attribute-outside-init +class AsyncProgressDownloader(JmAsyncDownloader): + + def start_progress(self, album_total=None, album_id=None): + from threading import Lock + + self.progress_lock = Lock() + self.album_total = album_total + self.album_id = album_id + self.album_done = 0 + self.chapter_done = {} + self.chapter_total = {} + self.chapter_tasks = {} + console = ProgressDownloader.get_progress_console() + self.progress = ProgressDownloader.new_rich_progress(console) + self.progress.start() + ProgressDownloader.register_progress(self.progress) + self.album_task = None + if album_total is not None: + self.album_task = self.progress.add_task( + f'[bold magenta]本子-{ProgressDownloader.display_id(album_id)}[/bold magenta]', + total=album_total, + ) + ProgressDownloader.refresh_progress(self) + + async def before_album(self, album): + await super().before_album(album) + self.start_progress(len(album), album.id) + + async def before_photo(self, photo): + await super().before_photo(photo) + if getattr(self, 'progress', None) is None: + self.start_progress() + with self.progress_lock: + self.chapter_done[photo.id] = 0 + self.chapter_total[photo.id] = len(photo) + self.chapter_tasks[photo.id] = self.progress.add_task( + f'[cyan]章节-{ProgressDownloader.display_id(photo.id)}[/cyan]', + total=len(photo), + ) + ProgressDownloader.refresh_progress(self) + + async def after_image(self, image, img_save_path): + await super().after_image(image, img_save_path) + photo_id = image.from_photo.id + with self.progress_lock: + self.chapter_done[photo_id] += 1 + self.progress.advance(self.chapter_tasks[photo_id]) + ProgressDownloader.refresh_progress(self) + + async def after_photo(self, photo): + await super().after_photo(photo) + with self.progress_lock: + done = self.chapter_done[photo.id] + succeeded = done == len(photo) + color = 'bold green' if succeeded else 'bold yellow' + icon = '✓' if succeeded else '⚠' + self.progress.update( + self.chapter_tasks[photo.id], + description=f'[{color}]{icon} 章节-{ProgressDownloader.display_id(photo.id)}[/{color}]', + ) + if succeeded: + self.album_done += 1 + if self.album_task is not None: + self.progress.advance(self.album_task) + ProgressDownloader.refresh_progress(self) + if self.album_total is None: + ProgressDownloader.print_non_interactive_summary(self, photo.id) + + async def after_album(self, album): + await super().after_album(album) + with self.progress_lock: + succeeded = self.album_done == len(album) + color = 'bold green' if succeeded else 'bold yellow' + icon = '✓' if succeeded else '⚠' + self.progress.update( + self.album_task, + description=f'[{color}]{icon} 本子-{ProgressDownloader.display_id(self.album_id)}[/{color}]', + ) + ProgressDownloader.refresh_progress(self) + ProgressDownloader.print_non_interactive_summary(self) + self.stop_progress() + + def stop_progress(self): + progress = getattr(self, 'progress', None) + if progress is None: + return + ProgressDownloader.unregister_progress(progress) + if progress.console.is_interactive: + progress.stop() + self.progress = None + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self.stop_progress() + return await super().__aexit__(exc_type, exc_val, exc_tb) + + +class DownloadProgressPlugin(JmOptionPlugin): + plugin_key = 'download_progress' + log_file = 'jmcomic-download.log' + + @staticmethod + def cli_no_progress_notice(): + if not get_jm_task_context().get('cli_no_progress'): + return '' + + return ( + '\n[bold yellow]⚠ 检测到命令行参数 --no-progress，' + '但是当前 Option 已配置 download_progress 插件，因此未关闭进度条。[/bold yellow]' + ) + + @classmethod + def build(cls, option): + plugin = cls(option) + try: + import rich + except ImportError: + plugin.warning_lib_not_install('rich') + plugin.log_path = plugin.redirect_log_to_file() + return plugin + + def redirect_log_to_file(self): + import logging + from pathlib import Path + from .jm_config import jm_logger + + class ProgressLogHandler(logging.Handler): + def emit(self, record): + try: + ProgressDownloader.append_progress_log(self.format(record)) + except Exception: + self.handleError(record) + + log_path = Path(self.log_file).resolve() + formatter = logging.Formatter( + '%(asctime)s [%(threadName)s] [%(topic)s] ' + '%(message)s' + ) + file_handler = logging.FileHandler(log_path, encoding='utf-8') + file_handler.setFormatter(formatter) + + for old_handler in jm_logger.handlers[:]: + jm_logger.removeHandler(old_handler) + + ProgressDownloader.reset_progress_logs() + jm_logger.addHandler(file_handler) + if ProgressDownloader.get_progress_console().is_interactive: + progress_handler = ProgressLogHandler() + progress_handler.setFormatter(formatter) + jm_logger.addHandler(progress_handler) + jm_logger.setLevel(logging.INFO) + jm_logger.propagate = False + return log_path + + @staticmethod + def print_non_interactive_notice(console, log_path): + from rich.panel import Panel + + console.print(Panel.fit( + '[bold green]✓ 下载进度插件已启用[/bold green]' + f'{DownloadProgressPlugin.cli_no_progress_notice()}\n\n' + '[cyan]显示模式[/cyan]：完成后汇总\n' + '[cyan]动态进度[/cyan]：请在 Terminal / PowerShell 中运行\n\n' + '[yellow]详细日志[/yellow]\n' + f'{log_path}', + title='[bold magenta]JMComic Progress[/bold magenta]', + border_style='bright_blue', + )) + + def invoke(self): + from rich.panel import Panel + + ProgressDownloader.use() + AsyncProgressDownloader.use() + self.log('已将默认 Downloader 替换为 ProgressDownloader') + self.log('已将默认 Async Downloader 替换为 AsyncProgressDownloader') + self.log(f'普通日志只写入文件: {self.log_path}') + console = ProgressDownloader.get_progress_console() + if console.is_interactive: + self.log('当前为交互终端，显示本子、章节两级彩色动态进度') + console.print(Panel.fit( + '[bold green]✓ 彩色下载进度插件已启用[/bold green]' + f'{self.cli_no_progress_notice()}\n' + '[cyan]Sync[/cyan]：ProgressDownloader\n' + '[cyan]Async[/cyan]：AsyncProgressDownloader\n' + f'[yellow]详细日志[/yellow]：{self.log_path}\n' + '[dim]终端只显示进度，普通日志不会刷屏[/dim]', + title='[bold magenta]JMComic Progress[/bold magenta]', + border_style='bright_blue', + )) + else: + self.log('当前不是交互终端，关闭动态进度，只在下载结束后输出汇总') + self.print_non_interactive_notice(console, self.log_path) + + class AutoSetBrowserCookiesPlugin(JmOptionPlugin): plugin_key = 'auto_set_browser_cookies' @@ -799,6 +1227,8 @@ def invoke(self, # noinspection PyTypeChecker detail: DetailEntity = album or photo + if downloader is not None: + downloader.record_export_filepath(detail, pdf_filepath) # 打印结果 self.log(f'{detail.alias_cn()}合并PDF成功！' @@ -881,6 +1311,8 @@ def invoke(self, return # noinspection PyTypeChecker detail: DetailEntity = album or photo + if downloader is not None: + downloader.record_export_filepath(detail, long_img_path) # 打印结果 self.log(f'{detail.alias_cn()}合并长图成功！' diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index 6b2b425bd..6dc8101e7 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -481,7 +481,7 @@ def _start_comment(self, attrs): comment = { 'CID': (attrs.get('data-cid') or '').strip('{}'), 'AID': None, - 'UID': None, + 'UID': attrs.get('data-userid') or None, 'parent_CID': parent['CID'] if parent is not None else None, 'content_parts': [], 'username': None, @@ -538,15 +538,22 @@ def handle_starttag(self, tag, attrs): if self._inside_class('timeline-left') and '/user/' in path: username = path.split('/user/', 1)[1].split('/', 1)[0] comment['username'] = unquote(username) - if self._inside_class('timeline-ft') and '/photo/' in path: - comment['AID'] = path.split('/photo/', 1)[1].split('/', 1)[0] + if self._inside_class('timeline-ft'): + if '/photo/' in path: + comment['AID'] = path.split('/photo/', 1)[1].split('/', 1)[0] + elif '/album/' in path: + comment['AID'] = path.split('/album/', 1)[1].split('/', 1)[0] if tag == 'img': path = urlparse(attrs.get('src') or '').path if self._inside_class('timeline-left') and '/media/users/' in path: + explicit_user_id = attrs.get('data-userid') + if explicit_user_id: + comment['UID'] = explicit_user_id + filename = path.split('/media/users/', 1)[1].split('/', 1)[0] user_id = filename.rsplit('.', 1)[0] - if user_id.isdigit(): + if comment['UID'] is None and user_id.isdigit(): comment['UID'] = user_id if tag in self.void_elements: @@ -655,7 +662,7 @@ class JmPageTool: ] @classmethod - def parse_html_to_search_page(cls, html: str) -> JmSearchPage: + def parse_html_to_search_page(cls, html: str, page_number: Optional[int] = None) -> JmSearchPage: # 1. 检查是否失败 PatternTool.require_not_match( html, @@ -684,10 +691,10 @@ def parse_html_to_search_page(cls, html: str) -> JmSearchPage: album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page )) - return JmSearchPage(content, total) + return JmSearchPage(content, total, page_number) @classmethod - def parse_html_to_category_page(cls, html: str) -> JmSearchPage: + def parse_html_to_category_page(cls, html: str, page_number: Optional[int] = None) -> JmSearchPage: content = [] total = int(PatternTool.match_or_default(html, *cls.pattern_html_search_total)) @@ -699,10 +706,10 @@ def parse_html_to_category_page(cls, html: str) -> JmSearchPage: album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page )) - return JmSearchPage(content, total) + return JmSearchPage(content, total, page_number) @classmethod - def parse_html_to_favorite_page(cls, html: str) -> JmFavoritePage: + def parse_html_to_favorite_page(cls, html: str, page_number: Optional[int] = None) -> JmFavoritePage: total = int(PatternTool.require_match( html, cls.pattern_html_favorite_total, @@ -722,10 +729,10 @@ def parse_html_to_favorite_page(cls, html: str) -> JmFavoritePage: folder_list_raw = p2.findall(folder_list_text) folder_list = [{'name': fname, 'FID': fid} for fid, fname in folder_list_raw] - return JmFavoritePage(content, folder_list, total) + return JmFavoritePage(content, folder_list, total, page_number) @classmethod - def parse_api_to_search_page(cls, data: AdvancedDict) -> JmSearchPage: + def parse_api_to_search_page(cls, data: AdvancedDict, page_number: Optional[int] = None) -> JmSearchPage: """ model_data: { "search_query": "MANA", @@ -751,10 +758,10 @@ def parse_api_to_search_page(cls, data: AdvancedDict) -> JmSearchPage: """ total: int = int(data.total or 0) # 2024.1.5 data.total可能为None content = cls.adapt_content(data.content) - return JmSearchPage(content, total) + return JmSearchPage(content, total, page_number) @classmethod - def parse_api_to_favorite_page(cls, data: AdvancedDict) -> JmFavoritePage: + def parse_api_to_favorite_page(cls, data: AdvancedDict, page_number: Optional[int] = None) -> JmFavoritePage: """ { "list": [ @@ -795,10 +802,13 @@ def parse_api_to_favorite_page(cls, data: AdvancedDict) -> JmFavoritePage: content = cls.adapt_content(data.list) folder_list = data.get('folder_list', []) - return JmFavoritePage(content, folder_list, total) + return JmFavoritePage(content, folder_list, total, page_number) @classmethod - def parse_api_to_album_comment_page(cls, data: AdvancedDict) -> JmAlbumCommentPage: + def parse_api_to_album_comment_page(cls, + data: AdvancedDict, + page_number=None, + ) -> JmAlbumCommentPage: def parse_comment(item): item_data = getattr(item, 'src_dict', item) or {} parser = HtmlTextParser() @@ -827,12 +837,20 @@ def parse_comment(item): return JmAlbumCommentPage( content=content, total=total, + page_number=page_number, raw_data=data, ) @classmethod - def parse_html_to_album_comment_page(cls, data: AdvancedDict) -> JmAlbumCommentPage: - raw_html = data.code or '' + def parse_html_to_album_comment_page(cls, + data: AdvancedDict, + page_number=None, + ) -> JmAlbumCommentPage: + raw_html = data.get('code') + if raw_html is None: + message = data.get('message', []) or [] + raw_html = ''.join(message) if isinstance(message, list) else message + parser = CommentParser() parser.feed(raw_html) parser.close() @@ -841,6 +859,7 @@ def parse_html_to_album_comment_page(cls, data: AdvancedDict) -> JmAlbumCommentP return JmAlbumCommentPage( content=content, total=None, + page_number=page_number, raw_html=raw_html, raw_data=data, ) diff --git a/tests/test_jmcomic/test_jm_async_client.py b/tests/test_jmcomic/test_jm_async_client.py index 5f0120b85..62093e089 100644 --- a/tests/test_jmcomic/test_jm_async_client.py +++ b/tests/test_jmcomic/test_jm_async_client.py @@ -83,6 +83,8 @@ async def run(): self.assertTrue(list(page_1)) self.assertTrue(list(page_2)) + self.assertEqual(page_1.page_number, 1) + self.assertEqual(page_2.page_number, 2) self.assertEqual(page_1.total, page_2.total) self.assertIsInstance(page_1.raw_data, AdvancedDict) self.assertGreaterEqual(page_1.comment_count, len(page_1)) @@ -91,11 +93,25 @@ async def run(): self.assertIsInstance(comment, JmAlbumComment) self.assertIsInstance(comment.raw_data, AdvancedDict) self.assertIsInstance(comment.is_spoiler, bool) + self.assertIn(str(comment.comment_id), str(comment)) + self.assertIn(comment.content, str(comment)) self.assertTrue(all( isinstance(reply, JmAlbumComment) for reply in comment.replies )) + forum_gen = self.async_client.forum_pagination_gen(page=1) + forum_page_1 = await forum_gen.__anext__() + forum_page_2 = await forum_gen.__anext__() + await forum_gen.aclose() + + self.assertTrue(list(forum_page_1)) + self.assertTrue(list(forum_page_2)) + self.assertEqual(forum_page_1.page_number, 1) + self.assertEqual(forum_page_2.page_number, 2) + self.assertGreater(forum_page_1.total, 0) + self.assertIsInstance(forum_page_1.raw_data, AdvancedDict) + self.run_async(run()) def test_async_album_pagination_without_total(self): @@ -254,9 +270,19 @@ def test_async_cache_on_off(self): """专门测试：async 缓存开启/关闭行为""" loop = asyncio.new_event_loop() client: AsyncJmcomicClient = self.option.new_jm_async_client() + album_detail_request_count = 0 try: loop.run_until_complete(client.setup()) + original_req_api = client.req_api + + async def counted_req_api(url, *args, **kwargs): + nonlocal album_detail_request_count + if url == client.API_ALBUM and kwargs.get('params') == {'id': '123'}: + album_detail_request_count += 1 + return await original_req_api(url, *args, **kwargs) + + client.req_api = counted_req_api # 1. 缓存默认关闭（_cache=None） self.assertIsNone(client.get_cache_dict(), '默认 cache 应为 None') @@ -264,8 +290,16 @@ def test_async_cache_on_off(self): # 开启缓存 client.set_cache_dict({}) album1 = loop.run_until_complete(client.get_album_detail('123')) + album1.save_path = '/tmp/album-123' + album1.duration = 1.0 album2 = loop.run_until_complete(client.get_album_detail('123')) - self.assertIs(album1, album2, '缓存开启：同 ID 应返回同一对象（对象引用相同）') + self.assertIsNot(album1, album2, '缓存开启：同 ID 应返回独立实体') + self.assertEqual(album1.id, album2.id, '缓存命中前后详情 ID 应一致') + self.assertEqual(album1.name, album2.name, '缓存命中前后详情名称应一致') + self.assertEqual(album1.tags, album2.tags, '缓存命中前后详情标签应一致') + self.assertEqual(album_detail_request_count, 1, '缓存命中时底层 album 详情请求应仅发生一次') + self.assertEqual(album2.save_path, '', '缓存模板不应携带上一次下载路径') + self.assertIsNone(album2.duration, '缓存模板不应携带上一次下载耗时') # 2. 关闭缓存 client.set_cache_dict(None) @@ -278,7 +312,8 @@ def test_async_cache_on_off(self): album4 = loop.run_until_complete(client.get_album_detail('123')) self.assertEqual(len(new_cache), 1, '新缓存应有 1 条记录') album5 = loop.run_until_complete(client.get_album_detail('123')) - self.assertIs(album4, album5, '重新开启缓存后应命中') + self.assertIsNot(album4, album5, '重新开启缓存后应返回独立实体') + self.assertEqual(album4.id, album5.id, '重新开启缓存后应命中相同详情数据') finally: loop.run_until_complete(client.close()) diff --git a/tests/test_jmcomic/test_jm_async_feature.py b/tests/test_jmcomic/test_jm_async_feature.py index 8b2992cf0..87e48c2e1 100644 --- a/tests/test_jmcomic/test_jm_async_feature.py +++ b/tests/test_jmcomic/test_jm_async_feature.py @@ -86,7 +86,7 @@ def test_async_export_album_use_photo_rule(self): sync_album = self.sync_api_client.get_album_detail(album_id) sync_raised = False try: - f.invoke(self.option, feature_from='download_album', when='after_album', + f.invoke(self.option, when='after_album', album=sync_album, photo=None) except AttributeError: sync_raised = True @@ -95,7 +95,7 @@ def test_async_export_album_use_photo_rule(self): async_album = self.run_async(self.async_client.get_album_detail(album_id)) async_raised = False try: - f.invoke(self.option, feature_from='download_album', when='after_album', + f.invoke(self.option, when='after_album', album=async_album, photo=None) except AttributeError: async_raised = True diff --git a/tests/test_jmcomic/test_jm_cli.py b/tests/test_jmcomic/test_jm_cli.py index bdbd913de..d20f16c1d 100644 --- a/tests/test_jmcomic/test_jm_cli.py +++ b/tests/test_jmcomic/test_jm_cli.py @@ -1,8 +1,10 @@ from test_jmcomic import * from io import StringIO -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import MagicMock, patch from jmcomic.cli import JmcomicUI, JmViewUI +from jmcomic.jm_task_context import get_jm_task_context class Test_Cli(JmTestConfigurable): @@ -10,13 +12,78 @@ class Test_Cli(JmTestConfigurable): album_id = '350234' - def test_cl_deprecated(self): - with self.assertWarnsRegex(DeprecationWarning, r'removed in version 2\.7\.4'): - from jmcomic.cl import JmcomicUI as DeprecatedJmcomicUI + # ========== jmcomic 命令测试 ========== - self.assertIs(DeprecatedJmcomicUI, JmcomicUI) + def test_jmcomic_progress_enabled_by_default(self): + ui = JmcomicUI() + with patch('sys.argv', ['jmcomic', self.album_id]): + ui.parse_arg() - # ========== jmcomic 命令测试 ========== + self.assertTrue(ui.progress_enabled) + + def test_jmcomic_no_progress_arg(self): + ui = JmcomicUI() + with patch('sys.argv', ['jmcomic', self.album_id, '--no-progress']): + ui.parse_arg() + + self.assertFalse(ui.progress_enabled) + + def test_jmcomic_enable_download_progress(self): + ui = JmcomicUI() + option = SimpleNamespace(plugins={}) + plugin = MagicMock() + + with patch('jmcomic.cli.importlib.util.find_spec', return_value=object()), \ + patch('jmcomic.jm_plugin.DownloadProgressPlugin.build', return_value=plugin) as build: + ui.enable_download_progress(option) + + build.assert_called_once_with(option) + plugin.invoke.assert_called_once_with() + + def test_jmcomic_does_not_enable_progress_twice(self): + ui = JmcomicUI() + option = SimpleNamespace(plugins={ + 'after_init': [{'plugin': 'download_progress'}] + }) + + with patch('jmcomic.jm_plugin.DownloadProgressPlugin.build') as build: + ui.enable_download_progress(option) + + build.assert_not_called() + + def test_jmcomic_no_progress_context_wraps_option_creation(self): + ui = JmcomicUI() + option = SimpleNamespace(plugins={}) + + def parse_arg(): + ui.progress_enabled = False + ui.option_path = None + + def create_default_option(): + self.assertTrue(get_jm_task_context().get('cli_no_progress')) + return option + + with patch.object(ui, 'parse_arg', side_effect=parse_arg), \ + patch('jmcomic.api.JmOption.default', side_effect=create_default_option), \ + patch.object(ui, 'enable_download_progress'), \ + patch.object(ui, 'run'), \ + patch('jmcomic.api.jm_log'): + ui.main() + + self.assertNotIn('cli_no_progress', get_jm_task_context()) + + def test_jmcomic_falls_back_without_rich(self): + ui = JmcomicUI() + option = SimpleNamespace(plugins={}) + + with patch('jmcomic.cli.importlib.util.find_spec', return_value=None), \ + patch('jmcomic.jm_config.jm_log') as jm_log: + ui.enable_download_progress(option) + + jm_log.assert_called_once_with( + 'command_line.progress', + '未安装 rich，继续使用普通日志。如需显示下载进度，请执行：pip install rich' + ) def test_jmcomic_parse_album_id(self): """jmcomic 解析 album id""" diff --git a/tests/test_jmcomic/test_jm_client.py b/tests/test_jmcomic/test_jm_client.py index 400721c71..e84814dc0 100644 --- a/tests/test_jmcomic/test_jm_client.py +++ b/tests/test_jmcomic/test_jm_client.py @@ -1,3 +1,6 @@ +import asyncio +from types import SimpleNamespace + from test_jmcomic import * @@ -255,6 +258,9 @@ def test_album_pagination(self): self.assertTrue(api_comments) self.assertTrue(list(api_page_2)) self.assertTrue(html_comments) + self.assertEqual(api_page.page_number, 1) + self.assertEqual(api_page_2.page_number, 2) + self.assertEqual(html_page.page_number, 1) self.assertIsInstance(api_page.raw_data, AdvancedDict) self.assertIsInstance(html_page.raw_data, AdvancedDict) self.assertGreaterEqual(api_page.comment_count, len(api_page)) @@ -277,6 +283,34 @@ def test_album_pagination(self): self.assertIsNone(html_page_without_total.page_count) self.assertIsNone(html_page_2_without_total.total) self.assertIsNone(html_page_2_without_total.page_count) + self.assertEqual(html_page_without_total.page_number, 1) + self.assertEqual(html_page_2_without_total.page_number, 2) + + api_forum_gen = api_client.forum_pagination_gen(page=1) + api_forum_page = next(api_forum_gen) + api_forum_page_2 = next(api_forum_gen) + api_forum_gen.close() + + html_forum_gen = html_client.forum_pagination_gen(page=1, with_ad_wcm=1) + html_forum_page = next(html_forum_gen) + html_forum_page_2 = next(html_forum_gen) + html_forum_gen.close() + + self.assertTrue(list(api_forum_page)) + self.assertTrue(list(api_forum_page_2)) + self.assertTrue(list(html_forum_page)) + self.assertTrue(list(html_forum_page_2)) + self.assertEqual(api_forum_page.page_number, 1) + self.assertEqual(api_forum_page_2.page_number, 2) + self.assertEqual(html_forum_page.page_number, 1) + self.assertEqual(html_forum_page_2.page_number, 2) + self.assertGreater(api_forum_page.total, 0) + self.assertIsNone(html_forum_page.total) + self.assertIsNone(html_forum_page.page_count) + self.assertIsInstance(api_forum_page.raw_data, AdvancedDict) + self.assertIsInstance(html_forum_page.raw_data, AdvancedDict) + self.assertTrue(any(comment.album_id for comment in html_forum_page)) + self.assertTrue(any(comment.user_id for comment in html_forum_page)) for comment in (api_comments[0], html_comments[0]): self.assertIsInstance(comment.raw_data, AdvancedDict) @@ -284,6 +318,8 @@ def test_album_pagination(self): self.assertEqual(str(comment.album_id), album_id) self.assertIsInstance(comment.content, str) self.assertIsInstance(comment.is_spoiler, bool) + self.assertIn(str(comment.comment_id), str(comment)) + self.assertIn(comment.content, str(comment)) api_comments_by_cid = { str(comment.comment_id): comment @@ -363,6 +399,34 @@ def test_album_pagination(self): self.assertTrue(checked_reply, 'API/HTML 前 5 页没有可对照的回评') + def test_html_forum_comment_id_parsing(self): + page = JmPageTool.parse_html_to_album_comment_page(AdvancedDict({ + 'code': ''' +
+
+ + + +
+
first comment
+
photo
+
+
+
+ +
+
second comment
+
album
+
+ ''', + }), page_number=1) + + self.assertEqual(page.page_number, 1) + self.assertEqual(page[0].user_id, '200') + self.assertEqual(page[0].album_id, '300') + self.assertEqual(page[1].user_id, '201') + self.assertEqual(page[1].album_id, '301') + def test_get_detail(self): client = self.client @@ -386,9 +450,12 @@ def test_cache_result_equal(self): for args in cases: photo = cl.get_photo_detail(*args) if ans is None: - ans = id(photo) + ans = photo else: - self.assertEqual(ans, id(photo)) + self.assertIsNot(ans, photo) + self.assertEqual(ans.id, photo.id) + self.assertEqual(ans.name, photo.name) + self.assertEqual(ans.tags, photo.tags) def test_search_generator(self): JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING = False @@ -404,23 +471,6 @@ def test_search_generator(self): break def test_cache_level(self): - def get(cl): - return cl.get_album_detail('123') - - def assertEqual(first_cl, second_cl, msg): - self.assertEqual( - get(first_cl), - get(second_cl), - msg, - ) - - def assertNotEqual(first_cl, second_cl, msg): - return self.assertNotEqual( - get(first_cl), - get(second_cl), - msg, - ) - cases = [ ( True, @@ -439,17 +489,22 @@ def run(arg1, arg2, arg3, arg4): c4 = op.new_jm_client(cache=arg4) c5 = op.new_jm_client(cache=False) - # c1 == c2 - # c3 == c4 - # c1 != c3 - assertEqual(c1, c2, 'equals in same option level') - assertNotEqual(c3, c4, 'not equals in client level') - assertNotEqual(c1, c3, 'not equals in different level') - - # c5 != c1, c2, c3, c4 - obj = get(c5) - self.assertNotEqual(obj, get(c1)) - self.assertNotEqual(obj, get(c3)) + self.assertIs( + c1.get_cache_dict(), + c2.get_cache_dict(), + 'clients in the same option level should share a cache dict', + ) + self.assertIsNot( + c3.get_cache_dict(), + c4.get_cache_dict(), + 'clients in the client level should use separate cache dicts', + ) + self.assertIsNot( + c1.get_cache_dict(), + c3.get_cache_dict(), + 'different cache levels should not share a cache dict', + ) + self.assertIsNone(c5.get_cache_dict(), 'cache=False should disable caching') for case in cases: run(*case) @@ -480,6 +535,49 @@ def test_search_advanced(self): self.print_page(page) break + def test_page_number(self): + search_page = JmPageTool.parse_api_to_search_page( + AdvancedDict.wrap({'total': '0', 'content': []}), + page_number=3, + ) + favorite_page = JmPageTool.parse_api_to_favorite_page( + AdvancedDict.wrap({'total': '0', 'list': [], 'folder_list': []}), + page_number=4, + ) + album = SimpleNamespace(album_id='123', name='album', tags=['tag']) + single_album_page = JmSearchPage.wrap_single_album(album, page_number=5) + + self.assertEqual(search_page.page_number, 3) + self.assertEqual(favorite_page.page_number, 4) + self.assertEqual(single_album_page.page_number, 5) + + comment_page = JmAlbumCommentPage([], 1, '', {'code': 'ok'}, page_number=6) + self.assertEqual(comment_page.raw_html, '') + self.assertEqual(comment_page.raw_data.code, 'ok') + self.assertEqual(comment_page.page_number, 6) + + def test_page_number_in_sync_generator(self): + def get_page(page): + return JmSearchPage([], 100, page) + + generator = JmcomicClient.do_page_iter(None, {}, 1, get_page) + + self.assertEqual(next(generator).page_number, 1) + self.assertEqual(generator.send({'page': 3}).page_number, 3) + + def test_page_number_in_async_generator(self): + async def run(): + async def get_page(page): + return JmSearchPage([], 100, page) + + generator = AsyncJmcomicClient.do_page_iter(None, {}, 1, get_page) + + self.assertEqual((await generator.asend(None)).page_number, 1) + self.assertEqual((await generator.asend({'page': 3})).page_number, 3) + await generator.aclose() + + asyncio.run(run()) + @staticmethod def print_page(page): # 打印page内容 diff --git a/tests/test_jmcomic/test_jm_download_manifest.py b/tests/test_jmcomic/test_jm_download_manifest.py new file mode 100644 index 000000000..48037c20b --- /dev/null +++ b/tests/test_jmcomic/test_jm_download_manifest.py @@ -0,0 +1,1052 @@ +import asyncio +import os +import sys +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import patch + +from test_jmcomic import * +from jmcomic.jm_async_client import AsyncJmApiClient +from jmcomic.jm_downloader import record_download_duration + + +def new_album_photo_images(image_count=1): + album = JmAlbumDetail( + album_id='123', + scramble_id='220980', + name='album', + episode_list=[('456', '1', 'photo')], + page_count=image_count, + pub_date='', + update_date='', + likes='0', + views='0', + comment_count=0, + works=[], + actors=[], + authors=['author'], + tags=['tag'], + ) + photo = JmPhotoDetail( + photo_id='456', + name='photo', + series_id='123', + sort=1, + scramble_id='220980', + page_arr=[f'{index:05}.jpg' for index in range(1, image_count + 1)], + data_original_domain='cdn.example', + from_album=album, + ) + return album, photo, list(photo) + + +class ContractOption: + + def __init__(self, base_dir): + self.base_dir = base_dir + self.plugin_event_list = [] + self.context_event_list = [] + self.after_image_callback = None + self.dir_rule = SimpleNamespace( + base_dir=base_dir, + decide_album_root_dir=lambda _album: os.path.join(base_dir, 'album'), + ) + self.download = SimpleNamespace( + threading=SimpleNamespace(image=1, photo=1), + ) + + def decide_image_save_dir(self, _photo): + return os.path.join(self.base_dir, 'album', 'photo') + + def decide_image_filepath(self, image): + return os.path.join(self.decide_image_save_dir(image.from_photo), image.filename) + + def decide_download_cache(self, _image): + return True + + def decide_download_image_decode(self, _image): + return False + + def decide_photo_batch_count(self, _album): + return 1 + + def decide_image_batch_count(self, _photo): + return 1 + + def call_all_plugin(self, group, **kwargs): + self.plugin_event_list.append((group, kwargs)) + self.context_event_list.append((group, get_jm_task_context())) + if group == 'after_image' and self.after_image_callback is not None: + self.after_image_callback(kwargs['image']) + + +class ContractSyncClient: + + def __init__(self, album, photo): + self.album = album + self.photo = photo + self.image_download_count = 0 + + def get_album_detail(self, _album_id): + return self.album + + def get_photo_detail(self, _photo_id): + return self.photo + + def check_photo(self, _photo): + return None + + def download_by_image_detail(self, _image, save_path, decode_image): + self.image_download_count += 1 + os.makedirs(os.path.dirname(save_path), exist_ok=True) + with open(save_path, 'wb') as f: + f.write(b'image') + + +class ContractSyncDownloader(JmDownloader): + + def __init__(self, option, album, photo, image_list): + self._contract_client = ContractSyncClient(album, photo) + self._contract_album = album + self._contract_photo = photo + self._contract_image_list = image_list + super().__init__(option) + + def create_client(self): + return self._contract_client + + def do_filter(self, detail): + if detail is self._contract_album: + return [self._contract_photo] + if detail is self._contract_photo: + return list(self._contract_image_list) + return detail + + def execute_on_condition(self, iter_objs, apply, count_batch): + for detail in self.do_filter(iter_objs): + apply(detail) + + +class ContractAsyncClient: + + def __init__(self, album, photo): + self.album = album + self.photo = photo + self.image_download_count = 0 + + async def get_album_detail(self, _album_id): + return self.album + + async def get_photo_detail(self, _photo_id): + return self.photo + + async def check_photo(self, _photo): + return None + + +class ContractAsyncDownloader(JmAsyncDownloader): + + def __init__(self, option, album, photo, image_list): + self._contract_album = album + self._contract_photo = photo + self._contract_image_list = image_list + super().__init__(option) + self.client = ContractAsyncClient(album, photo) + + def do_filter(self, detail): + if detail is self._contract_album: + return [self._contract_photo] + if detail is self._contract_photo: + return list(self._contract_image_list) + return detail + + +class Test_Download_Manifest(unittest.TestCase): + + def test_downloadable_defaults(self): + album, photo, image_list = new_album_photo_images() + + for detail in (album, photo, image_list[0]): + self.assertEqual(detail.save_path, '') + self.assertFalse(detail.exists) + self.assertFalse(detail.skip) + self.assertTrue(detail.cache) + self.assertIsNone(detail.duration) + + def test_manifest_containers_are_not_shared(self): + first = DownloadManifest() + second = DownloadManifest() + + first.image_filepath_list.append('/tmp/1.jpg') + first.export_filepath_dict['pdf'] = ['/tmp/1.pdf'] + + self.assertEqual(second.image_filepath_list, []) + self.assertEqual(second.export_filepath_dict, {}) + + def test_export_filepath_lookup_normalizes_suffix(self): + manifest = DownloadManifest() + manifest.export_filepath_dict['pdf'] = ['/tmp/1.pdf'] + + self.assertEqual(manifest.get_export_filepath_list('pdf'), ['/tmp/1.pdf']) + self.assertEqual(manifest.get_export_filepath_list('.PDF'), ['/tmp/1.pdf']) + self.assertEqual(manifest.get_export_filepath_list('zip'), []) + + def test_download_result_remains_a_two_item_tuple(self): + album, _, _ = new_album_photo_images() + option = ContractOption('/tmp') + downloader = BaseDownloader(option) + manifest = DownloadManifest() + downloader.manifest_dict[album] = manifest + + result = DownloadResult(album, downloader) + unpacked_album, unpacked_downloader = result + + self.assertIsInstance(result, tuple) + self.assertEqual(len(result), 2) + self.assertIs(unpacked_album, album) + self.assertIs(unpacked_downloader, downloader) + self.assertIs(result.manifest, manifest) + self.assertIsNone(result.duration) + + album.duration = 1.25 + self.assertIsNone(result.duration) + + manifest.duration = 2.5 + + self.assertEqual(result.duration, 2.5) + + def test_duration_decorator_accepts_keyword_entity_arguments(self): + album, photo, _ = new_album_photo_images() + sync_times = iter((10.0, 12.5)) + async_times = iter((20.0, 24.0)) + + class Downloader: + + @record_download_duration('album_started_at', clock=lambda: next(sync_times)) + def download_album(self, album_id): + return album + + @record_download_duration('photo_started_at', clock=lambda: next(async_times)) + async def download_photo(self, photo_id): + return photo + + downloader = Downloader() + + self.assertIs(downloader.download_album(album_id='123'), album) + self.assertEqual(album.duration, 2.5) + self.assertIs(asyncio.run(downloader.download_photo(photo_id='456')), photo) + self.assertEqual(photo.duration, 4.0) + + def test_export_plugins_allow_omitting_downloader(self): + _, photo, _ = new_album_photo_images() + option = ContractOption('/tmp') + + cases = ( + (Img2pdfPlugin, 'img2pdf', 'decide_filepath', 'write_img_2_pdf', '/tmp/photo.pdf', (['/tmp/1.jpg'], ['/tmp/photo'])), + (LongImgPlugin, 'PIL', 'decide_filepath', 'write_img_2_long_img', '/tmp/photo.png', ['/tmp/1.jpg']), + ) + for plugin_class, module_name, filepath_method, write_method, output_path, write_result in cases: + with self.subTest(plugin=plugin_class.plugin_key): + plugin = plugin_class(option) + fake_module = SimpleNamespace(Image=object()) + with patch.dict(sys.modules, {module_name: fake_module}), \ + patch.object(plugin, filepath_method, return_value=output_path), \ + patch.object(plugin, write_method, return_value=write_result), \ + patch.object(plugin, 'log'): + plugin.invoke(photo=photo) + + def test_record_export_filepath_uses_top_level_album_manifest(self): + album, photo, _ = new_album_photo_images() + downloader = BaseDownloader(ContractOption('/tmp')) + manifest = downloader.begin_manifest(album) + + downloader.record_export_filepath(photo, '/tmp/album.PDF') + + self.assertIs(downloader.finish_manifest(album), manifest) + self.assertEqual(manifest.export_filepath_dict, {'pdf': ['/tmp/album.PDF']}) + + def test_record_export_filepath_uses_top_level_photo_manifest(self): + _, photo, _ = new_album_photo_images() + downloader = BaseDownloader(ContractOption('/tmp')) + manifest = downloader.begin_manifest(photo) + + downloader.record_export_filepath(photo, '/tmp/photo.zip') + + self.assertIs(downloader.finish_manifest(photo), manifest) + self.assertEqual(manifest.export_filepath_dict, {'zip': ['/tmp/photo.zip']}) + + + @staticmethod + def new_manifest_downloader(base_dir, top_level='album'): + album, photo, image_list = new_album_photo_images() + option = ContractOption(base_dir) + downloader = BaseDownloader(option) + if top_level == 'album': + downloader.begin_manifest(album) + else: + downloader.begin_manifest(photo) + + downloader.download_success_dict[album] = { + photo: [ + (option.decide_image_filepath(image), image) + for image in image_list + ] + } + return album, photo, image_list, downloader + + def test_record_export_filepath_groups_real_suffixes(self): + with TemporaryDirectory() as temp_dir: + album, photo, _, downloader = self.new_manifest_downloader(temp_dir) + cbz_path = os.path.join(temp_dir, 'album.cbz') + pdf_path = os.path.join(temp_dir, 'photo.PDF') + png_path = os.path.join(temp_dir, 'photo.png') + + downloader.record_export_filepath(album, cbz_path) + downloader.record_export_filepath(photo, pdf_path) + downloader.record_export_filepath(photo, png_path) + + manifest = downloader.manifest_dict[album] + self.assertEqual(manifest.export_filepath_dict, { + 'cbz': [cbz_path], + 'pdf': [pdf_path], + 'png': [png_path], + }) + + def test_record_export_filepath_rejects_missing_manifest(self): + with TemporaryDirectory() as temp_dir: + album, _, _ = new_album_photo_images() + downloader = BaseDownloader(ContractOption(temp_dir)) + + with self.assertRaisesRegex(JmcomicException, '没有活动的下载清单'): + downloader.record_export_filepath(album, os.path.join(temp_dir, 'album.zip')) + + def test_record_export_filepath_ignores_paths_without_suffix(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + manifest = downloader.manifest_dict[album] + + downloader.record_export_filepath(album, os.path.join(temp_dir, 'export')) + + self.assertEqual(manifest.export_filepath_dict, {}) + + def test_zip_album_registers_configured_suffix_once_after_success(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = ZipPlugin(downloader.option) + zip_dir = os.path.join(temp_dir, 'exports') + expected_path = plugin.decide_filepath(album, None, 'Aid', 'cbz', os.path.abspath(zip_dir), None) + order = [] + original_record = downloader.record_export_filepath + + def record(detail, filepath): + order.append(('record', filepath)) + return original_record(detail, filepath) + + def fake_zip_album(*args, **kwargs): + order.append(('zip_album', expected_path)) + + def fake_after_zip(_paths): + order.append(('after_zip', None)) + self.assertEqual( + downloader.manifest_dict[album].export_filepath_dict, + {'cbz': [expected_path]}, + ) + + downloader.record_export_filepath = record + + with patch.object(plugin, 'zip_album', side_effect=fake_zip_album), \ + patch.object(plugin, 'after_zip', side_effect=fake_after_zip): + plugin.invoke( + downloader=downloader, + album=album, + filename_rule='Aid', + suffix='cbz', + zip_dir=zip_dir, + ) + + self.assertEqual(order, [ + ('zip_album', expected_path), + ('record', expected_path), + ('after_zip', None), + ]) + + def test_zip_photo_registers_to_album_manifest_once_after_success(self): + with TemporaryDirectory() as temp_dir: + album, photo, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = ZipPlugin(downloader.option) + zip_dir = os.path.join(temp_dir, 'exports') + expected_path = plugin.decide_filepath(photo.from_album, photo, 'Pid', 'cbz', os.path.abspath(zip_dir), None) + order = [] + original_record = downloader.record_export_filepath + + def record(detail, filepath): + order.append(('record', detail, filepath)) + return original_record(detail, filepath) + + def fake_zip_photo(current_photo, image_list, zip_path, path_to_delete, encrypt): + self.assertIs(current_photo, photo) + self.assertEqual(zip_path, expected_path) + order.append(('zip_photo', current_photo, zip_path)) + + def fake_after_zip(_paths): + order.append(('after_zip', None, None)) + self.assertEqual( + downloader.manifest_dict[album].export_filepath_dict, + {'cbz': [expected_path]}, + ) + + downloader.record_export_filepath = record + + with patch.object(plugin, 'zip_photo', side_effect=fake_zip_photo), \ + patch.object(plugin, 'after_zip', side_effect=fake_after_zip): + plugin.invoke( + downloader=downloader, + album=album, + level='photo', + filename_rule='Pid', + suffix='cbz', + zip_dir=zip_dir, + ) + + self.assertEqual(order, [ + ('zip_photo', photo, expected_path), + ('record', photo, expected_path), + ('after_zip', None, None), + ]) + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict['cbz'].count(expected_path), 1) + + def test_pdf_registers_once_before_optional_deletion(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = Img2pdfPlugin(downloader.option) + expected_path = plugin.decide_filepath(album, None, 'Aid', 'pdf', temp_dir, None) + order = [] + original_record = downloader.record_export_filepath + + def record(detail, filepath): + order.append(('record', filepath)) + return original_record(detail, filepath) + + def fake_write(pdf_filepath, current_album, current_photo, encrypt): + self.assertEqual(pdf_filepath, expected_path) + self.assertIs(current_album, album) + self.assertIsNone(current_photo) + order.append(('write', pdf_filepath)) + return ['img1.jpg'], ['photo_dir'] + + def fake_delete(paths): + order.append(('delete', list(paths))) + self.assertEqual( + downloader.manifest_dict[album].export_filepath_dict, + {'pdf': [expected_path]}, + ) + + downloader.record_export_filepath = record + + with patch.dict(sys.modules, {'img2pdf': object()}), \ + patch.object(plugin, 'write_img_2_pdf', side_effect=fake_write), \ + patch.object(plugin, 'execute_deletion', side_effect=fake_delete): + plugin.invoke( + album=album, + downloader=downloader, + pdf_dir=temp_dir, + filename_rule='Aid', + ) + + self.assertEqual(order[0], ('write', expected_path)) + self.assertEqual(order[1], ('record', expected_path)) + self.assertEqual(order[2], ('delete', ['img1.jpg', 'photo_dir'])) + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict['pdf'].count(expected_path), 1) + + def test_pdf_photo_level_registers_photo_detail_once(self): + with TemporaryDirectory() as temp_dir: + _, photo, _, downloader = self.new_manifest_downloader(temp_dir, top_level='photo') + plugin = Img2pdfPlugin(downloader.option) + expected_path = plugin.decide_filepath(None, photo, 'Pid', 'pdf', temp_dir, None) + order = [] + original_record = downloader.record_export_filepath + + def record(detail, filepath): + order.append(('record', detail, filepath)) + return original_record(detail, filepath) + + def fake_write(pdf_filepath, current_album, current_photo, encrypt): + self.assertEqual(pdf_filepath, expected_path) + self.assertIsNone(current_album) + self.assertIs(current_photo, photo) + order.append(('write', current_photo, pdf_filepath)) + return ['img1.jpg'], ['photo_dir'] + + def fake_delete(paths): + order.append(('delete', list(paths))) + self.assertEqual( + downloader.manifest_dict[photo].export_filepath_dict, + {'pdf': [expected_path]}, + ) + + downloader.record_export_filepath = record + + with patch.dict(sys.modules, {'img2pdf': object()}), \ + patch.object(plugin, 'write_img_2_pdf', side_effect=fake_write), \ + patch.object(plugin, 'execute_deletion', side_effect=fake_delete): + plugin.invoke( + photo=photo, + downloader=downloader, + pdf_dir=temp_dir, + filename_rule='Pid', + ) + + self.assertEqual(order[0], ('write', photo, expected_path)) + self.assertEqual(order[1], ('record', photo, expected_path)) + self.assertEqual(order[2], ('delete', ['img1.jpg', 'photo_dir'])) + self.assertEqual(downloader.manifest_dict[photo].export_filepath_dict['pdf'].count(expected_path), 1) + + def test_pdf_missing_library_registers_nothing(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = Img2pdfPlugin(downloader.option) + original_import = __import__ + + def fake_import(name, *args, **kwargs): + if name == 'img2pdf': + raise ImportError('img2pdf missing') + return original_import(name, *args, **kwargs) + + with patch('builtins.__import__', side_effect=fake_import): + with self.assertRaises(PluginValidationException): + plugin.invoke( + album=album, + downloader=downloader, + pdf_dir=temp_dir, + filename_rule='Aid', + ) + + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict, {}) + + def test_pdf_empty_source_registers_nothing(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = Img2pdfPlugin(downloader.option) + + with patch.dict(sys.modules, {'img2pdf': object()}), \ + patch.object(plugin, 'write_img_2_pdf', return_value=None), \ + patch.object(plugin, 'execute_deletion') as delete_mock: + plugin.invoke( + album=album, + downloader=downloader, + pdf_dir=temp_dir, + filename_rule='Aid', + ) + + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict, {}) + delete_mock.assert_not_called() + + def test_long_img_registers_once_before_optional_deletion(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = LongImgPlugin(downloader.option) + expected_path = plugin.decide_filepath(album, None, 'Aid', 'png', temp_dir, None) + order = [] + original_record = downloader.record_export_filepath + + def record(detail, filepath): + order.append(('record', filepath)) + return original_record(detail, filepath) + + def fake_write(long_img_path, current_album, current_photo): + self.assertEqual(long_img_path, expected_path) + self.assertIs(current_album, album) + self.assertIsNone(current_photo) + order.append(('write', long_img_path)) + return ['img1.jpg'] + + def fake_delete(paths): + order.append(('delete', list(paths))) + self.assertEqual( + downloader.manifest_dict[album].export_filepath_dict, + {'png': [expected_path]}, + ) + + downloader.record_export_filepath = record + + with patch.dict(sys.modules, {'PIL': SimpleNamespace(Image=object())}), \ + patch.object(plugin, 'write_img_2_long_img', side_effect=fake_write), \ + patch.object(plugin, 'execute_deletion', side_effect=fake_delete): + plugin.invoke( + album=album, + downloader=downloader, + img_dir=temp_dir, + filename_rule='Aid', + ) + + self.assertEqual(order[0], ('write', expected_path)) + self.assertEqual(order[1], ('record', expected_path)) + self.assertEqual(order[2], ('delete', ['img1.jpg'])) + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict['png'].count(expected_path), 1) + + def test_long_img_photo_level_registers_photo_detail_once(self): + with TemporaryDirectory() as temp_dir: + _, photo, _, downloader = self.new_manifest_downloader(temp_dir, top_level='photo') + plugin = LongImgPlugin(downloader.option) + expected_path = plugin.decide_filepath(None, photo, 'Pid', 'png', temp_dir, None) + order = [] + original_record = downloader.record_export_filepath + + def record(detail, filepath): + order.append(('record', detail, filepath)) + return original_record(detail, filepath) + + def fake_write(long_img_path, current_album, current_photo): + self.assertEqual(long_img_path, expected_path) + self.assertIsNone(current_album) + self.assertIs(current_photo, photo) + order.append(('write', current_photo, long_img_path)) + return ['img1.jpg'] + + def fake_delete(paths): + order.append(('delete', list(paths))) + self.assertEqual( + downloader.manifest_dict[photo].export_filepath_dict, + {'png': [expected_path]}, + ) + + downloader.record_export_filepath = record + + with patch.dict(sys.modules, {'PIL': SimpleNamespace(Image=object())}), \ + patch.object(plugin, 'write_img_2_long_img', side_effect=fake_write), \ + patch.object(plugin, 'execute_deletion', side_effect=fake_delete): + plugin.invoke( + photo=photo, + downloader=downloader, + img_dir=temp_dir, + filename_rule='Pid', + ) + + self.assertEqual(order[0], ('write', photo, expected_path)) + self.assertEqual(order[1], ('record', photo, expected_path)) + self.assertEqual(order[2], ('delete', ['img1.jpg'])) + self.assertEqual(downloader.manifest_dict[photo].export_filepath_dict['png'].count(expected_path), 1) + + def test_long_img_missing_library_registers_nothing(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = LongImgPlugin(downloader.option) + original_import = __import__ + + def fake_import(name, *args, **kwargs): + if name == 'PIL': + raise ImportError('PIL missing') + return original_import(name, *args, **kwargs) + + with patch('builtins.__import__', side_effect=fake_import): + with self.assertRaises(PluginValidationException): + plugin.invoke( + album=album, + downloader=downloader, + img_dir=temp_dir, + filename_rule='Aid', + ) + + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict, {}) + + def test_long_img_empty_source_registers_nothing(self): + with TemporaryDirectory() as temp_dir: + album, _, _, downloader = self.new_manifest_downloader(temp_dir) + plugin = LongImgPlugin(downloader.option) + + with patch.dict(sys.modules, {'PIL': SimpleNamespace(Image=object())}), \ + patch.object(plugin, 'write_img_2_long_img', return_value=None), \ + patch.object(plugin, 'execute_deletion') as delete_mock: + plugin.invoke( + album=album, + downloader=downloader, + img_dir=temp_dir, + filename_rule='Aid', + ) + + self.assertEqual(downloader.manifest_dict[album].export_filepath_dict, {}) + delete_mock.assert_not_called() + + + def test_sync_detail_cache_returns_clean_copy(self): + class CachedClient(AbstractJmClient): + func_to_cache = ['fetch_detail_entity'] + + def fetch_detail_entity(self, _album_id): + self.fetch_count += 1 + album, _, _ = new_album_photo_images() + return album + + client = object.__new__(CachedClient) + client.CLIENT_CACHE = {} + client.fetch_count = 0 + client.enable_cache() + + first = client.fetch_detail_entity('123') + first.save_path = '/download/123' + first.duration = 1.5 + first.tags.append('mutated') + second = client.fetch_detail_entity('123') + + self.assertEqual(client.fetch_count, 1) + self.assertIsNot(first, second) + self.assertEqual(first.id, second.id) + self.assertEqual(second.save_path, '') + self.assertIsNone(second.duration) + self.assertNotIn('mutated', second.tags) + + def test_async_detail_cache_returns_clean_copy(self): + async def run_test(): + client = object.__new__(AsyncJmApiClient) + client._cache = {} + client.request_count = 0 + + class Response: + encoded_data = 'encoded' + res_data = { + 'id': '123', + 'name': 'album', + 'author': ['author'], + 'images': [], + 'description': '', + 'total_views': '0', + 'likes': '0', + 'series': [], + 'comment_total': '0', + 'tags': ['tag'], + 'works': [], + 'actors': [], + 'related_list': [], + } + + async def req_api(_url, **_kwargs): + client.request_count += 1 + return Response() + + client.req_api = req_api + + first = await client.get_album_detail('123') + first.save_path = '/download/123' + first.duration = 1.5 + first.tags.append('mutated') + second = await client.get_album_detail('123') + + self.assertEqual(client.request_count, 1) + self.assertIsNot(first, second) + self.assertEqual(first.id, second.id) + self.assertEqual(second.save_path, '') + self.assertIsNone(second.duration) + self.assertNotIn('mutated', second.tags) + + asyncio.run(run_test()) + + + def new_downloader(self, base_dir, image_count=1): + album, photo, image_list = new_album_photo_images(image_count) + option = ContractOption(base_dir) + downloader = ContractSyncDownloader(option, album, photo, image_list) + return album, photo, image_list, option, downloader + + @staticmethod + def create_cached_images(option, image_list): + for image in image_list: + filepath = option.decide_image_filepath(image) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, 'wb') as f: + f.write(b'cached') + + def test_album_photo_image_paths_and_durations(self): + with TemporaryDirectory() as temp_dir: + album, photo, image_list, option, downloader = self.new_downloader(temp_dir) + self.create_cached_images(option, image_list) + + downloader.download_album(album.id) + + self.assertEqual(album.save_path, option.dir_rule.decide_album_root_dir(album)) + self.assertEqual(photo.save_path, option.decide_image_save_dir(photo)) + self.assertEqual(image_list[0].save_path, option.decide_image_filepath(image_list[0])) + self.assertIsInstance(album.duration, float) + self.assertIsInstance(photo.duration, float) + self.assertIsInstance(image_list[0].duration, float) + + def test_entity_timing_contexts_are_nested_and_do_not_leak(self): + with TemporaryDirectory() as temp_dir: + album, photo, image_list, option, downloader = self.new_downloader(temp_dir, image_count=2) + self.create_cached_images(option, image_list) + + with patch('jmcomic.jm_downloader.perf_counter', side_effect=range(1, 9)): + downloader.download_album(album.id) + + context_by_group = {} + for group, context in option.context_event_list: + context_by_group.setdefault(group, []).append(context) + + self.assertEqual(1, context_by_group['before_album'][0].get('album_started_at')) + self.assertEqual( + {'album_started_at': 1, 'photo_started_at': 2}, + context_by_group['before_photo'][0], + ) + self.assertEqual( + [3, 5], + [context['image_started_at'] for context in context_by_group['before_image']], + ) + self.assertEqual( + [2, 2], + [context['photo_started_at'] for context in context_by_group['before_image']], + ) + self.assertEqual({}, get_jm_task_context()) + + def test_sync_download_album_duration_includes_detail_and_manifest(self): + clock = {'now': 10.0} + contexts = [] + album = SimpleNamespace(duration=None) + downloader = object.__new__(JmDownloader) + + def get_album_detail(_album_id): + contexts.append(get_jm_task_context()) + clock['now'] = 20.0 + return album + + def begin_manifest(_album): + contexts.append(get_jm_task_context()) + clock['now'] = 30.0 + + def download_by_album_detail(_album): + contexts.append(get_jm_task_context()) + clock['now'] = 40.0 + + def finish_manifest(_album): + contexts.append(get_jm_task_context()) + clock['now'] = 50.0 + + downloader.client = SimpleNamespace(get_album_detail=get_album_detail) + downloader.begin_manifest = begin_manifest + downloader.download_by_album_detail = download_by_album_detail + downloader.finish_manifest = finish_manifest + + with patch('jmcomic.jm_downloader.perf_counter', side_effect=lambda: clock['now']): + result = JmDownloader.download_album(downloader, '123') + + self.assertIs(result, album) + self.assertEqual(40.0, album.duration) + self.assertEqual([10.0] * 4, [context.get('album_started_at') for context in contexts]) + self.assertEqual({}, get_jm_task_context()) + + def test_sync_download_photo_duration_includes_detail_and_manifest(self): + clock = {'now': 10.0} + contexts = [] + photo = SimpleNamespace(duration=None) + downloader = object.__new__(JmDownloader) + + def get_photo_detail(_photo_id): + contexts.append(get_jm_task_context()) + clock['now'] = 20.0 + return photo + + def begin_manifest(_photo): + contexts.append(get_jm_task_context()) + clock['now'] = 30.0 + + def download_by_photo_detail(_photo): + contexts.append(get_jm_task_context()) + clock['now'] = 40.0 + + def finish_manifest(_photo): + contexts.append(get_jm_task_context()) + clock['now'] = 50.0 + + downloader.client = SimpleNamespace(get_photo_detail=get_photo_detail) + downloader.begin_manifest = begin_manifest + downloader.download_by_photo_detail = download_by_photo_detail + downloader.finish_manifest = finish_manifest + + with patch('jmcomic.jm_downloader.perf_counter', side_effect=lambda: clock['now']): + result = JmDownloader.download_photo(downloader, '456') + + self.assertIs(result, photo) + self.assertEqual(40.0, photo.duration) + self.assertEqual([10.0] * 4, [context.get('photo_started_at') for context in contexts]) + self.assertEqual({}, get_jm_task_context()) + + def test_async_download_album_duration_includes_detail_and_manifest(self): + async def run_test(): + clock = {'now': 10.0} + contexts = [] + album = SimpleNamespace(duration=None) + downloader = object.__new__(JmAsyncDownloader) + + async def get_album_detail(_album_id): + contexts.append(get_jm_task_context()) + clock['now'] = 20.0 + return album + + def begin_manifest(_album): + contexts.append(get_jm_task_context()) + clock['now'] = 30.0 + + async def download_by_album_detail(_album): + contexts.append(get_jm_task_context()) + clock['now'] = 40.0 + + def finish_manifest(_album): + contexts.append(get_jm_task_context()) + clock['now'] = 50.0 + + downloader.client = SimpleNamespace(get_album_detail=get_album_detail) + downloader.begin_manifest = begin_manifest + downloader.download_by_album_detail = download_by_album_detail + downloader.finish_manifest = finish_manifest + + with patch('jmcomic.jm_downloader.perf_counter', side_effect=lambda: clock['now']): + result = await JmAsyncDownloader.download_album(downloader, '123') + + self.assertIs(result, album) + self.assertEqual(40.0, album.duration) + self.assertEqual([10.0] * 4, [context.get('album_started_at') for context in contexts]) + self.assertEqual({}, get_jm_task_context()) + + asyncio.run(run_test()) + + def test_async_download_photo_duration_includes_detail_and_manifest(self): + async def run_test(): + clock = {'now': 10.0} + contexts = [] + photo = SimpleNamespace(duration=None) + downloader = object.__new__(JmAsyncDownloader) + + async def get_photo_detail(_photo_id): + contexts.append(get_jm_task_context()) + clock['now'] = 20.0 + return photo + + def begin_manifest(_photo): + contexts.append(get_jm_task_context()) + clock['now'] = 30.0 + + async def download_by_photo_detail(_photo): + contexts.append(get_jm_task_context()) + clock['now'] = 40.0 + + def finish_manifest(_photo): + contexts.append(get_jm_task_context()) + clock['now'] = 50.0 + + downloader.client = SimpleNamespace(get_photo_detail=get_photo_detail) + downloader.begin_manifest = begin_manifest + downloader.download_by_photo_detail = download_by_photo_detail + downloader.finish_manifest = finish_manifest + + with patch('jmcomic.jm_downloader.perf_counter', side_effect=lambda: clock['now']): + result = await JmAsyncDownloader.download_photo(downloader, '456') + + self.assertIs(result, photo) + self.assertEqual(40.0, photo.duration) + self.assertEqual([10.0] * 4, [context.get('photo_started_at') for context in contexts]) + self.assertEqual({}, get_jm_task_context()) + + asyncio.run(run_test()) + + def test_cache_hit_triggers_after_image_and_success_record(self): + with TemporaryDirectory() as temp_dir: + album, photo, image_list, option, downloader = self.new_downloader(temp_dir) + self.create_cached_images(option, image_list) + + downloader.download_album(album.id) + + after_image_events = [event for event, _ in option.plugin_event_list if event == 'after_image'] + self.assertEqual(after_image_events, ['after_image']) + self.assertEqual(downloader._contract_client.image_download_count, 0) + self.assertEqual( + downloader.download_success_dict[album][photo], + [(image_list[0].save_path, image_list[0])], + ) + + def test_album_manifest_collects_cached_images_in_entity_order(self): + with TemporaryDirectory() as temp_dir: + album, _, image_list, option, downloader = self.new_downloader(temp_dir, image_count=2) + self.create_cached_images(option, image_list) + + downloader.download_album(album.id) + + manifest = downloader.manifest_dict[album] + self.assertEqual( + manifest.image_filepath_list, + [image.save_path for image in image_list], + ) + + def test_album_internal_photo_does_not_create_another_manifest(self): + with TemporaryDirectory() as temp_dir: + album, photo, image_list, option, downloader = self.new_downloader(temp_dir) + self.create_cached_images(option, image_list) + + downloader.download_album(album.id) + + self.assertEqual(set(downloader.manifest_dict), {album}) + self.assertNotIn(photo, downloader.manifest_dict) + + def test_top_level_photo_creates_photo_manifest(self): + with TemporaryDirectory() as temp_dir: + album, photo, image_list, option, downloader = self.new_downloader(temp_dir) + self.create_cached_images(option, image_list) + + downloaded_photo = downloader.download_photo(photo.id) + + self.assertIs(downloaded_photo, photo) + self.assertEqual(set(downloader.manifest_dict), {photo}) + self.assertEqual( + downloader.manifest_dict[photo].image_filepath_list, + [image.save_path for image in image_list], + ) + + def test_skipped_image_has_duration_but_is_not_in_manifest(self): + with TemporaryDirectory() as temp_dir: + album, _, image_list, option, downloader = self.new_downloader(temp_dir) + image_list[0].skip = True + + downloader.download_album(album.id) + + self.assertIsInstance(image_list[0].duration, float) + self.assertEqual(downloader.manifest_dict[album].image_filepath_list, []) + + def test_manifest_uses_final_path_after_image_plugin(self): + with TemporaryDirectory() as temp_dir: + album, _, image_list, option, downloader = self.new_downloader(temp_dir) + self.create_cached_images(option, image_list) + final_path = os.path.join(temp_dir, 'converted', '00001.webp') + option.after_image_callback = lambda image: setattr(image, 'save_path', final_path) + + downloader.download_album(album.id) + + self.assertEqual(image_list[0].save_path, final_path) + self.assertEqual(downloader.manifest_dict[album].image_filepath_list, [final_path]) + + + def test_cache_hit_paths_durations_after_image_and_manifest(self): + async def run_test(temp_dir): + album, photo, image_list = new_album_photo_images() + option = ContractOption(temp_dir) + image = image_list[0] + filepath = option.decide_image_filepath(image) + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, 'wb') as f: + f.write(b'cached') + + downloader = ContractAsyncDownloader(option, album, photo, image_list) + try: + await downloader.download_album(album.id) + + self.assertEqual(album.save_path, option.dir_rule.decide_album_root_dir(album)) + self.assertEqual(photo.save_path, option.decide_image_save_dir(photo)) + self.assertEqual(image.save_path, filepath) + self.assertIsInstance(album.duration, float) + self.assertIsInstance(photo.duration, float) + self.assertIsInstance(image.duration, float) + after_image_events = [event for event, _ in option.plugin_event_list if event == 'after_image'] + self.assertEqual(after_image_events, ['after_image']) + self.assertEqual(downloader.download_success_dict[album][photo], [(filepath, image)]) + self.assertEqual(downloader.manifest_dict[album].image_filepath_list, [filepath]) + finally: + downloader.shutdown() + + with TemporaryDirectory() as temp_dir: + asyncio.run(run_test(temp_dir)) diff --git a/tests/test_jmcomic/test_jm_download_progress.py b/tests/test_jmcomic/test_jm_download_progress.py new file mode 100644 index 000000000..02cbd6191 --- /dev/null +++ b/tests/test_jmcomic/test_jm_download_progress.py @@ -0,0 +1,525 @@ +import contextlib +import asyncio +import importlib.util +import logging +import os +import time +import unittest +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from jmcomic import ( + AsyncProgressDownloader, + DownloadProgressPlugin, + JmAlbumDetail, + JmAsyncDownloader, + JmDownloader, + JmModuleConfig, + ProgressDownloader, + create_option_by_str, + download_album, + download_album_async, + jm_logger, + new_async_downloader, +) +from jmcomic.jm_config import setup_default_jm_logger +from jmcomic.jm_task_context import jm_task_context + + +PROJECT_DIR = Path(__file__).resolve().parents[2] +DOCUMENT_FILE = PROJECT_DIR / 'assets' / 'docs' / 'sources' / 'tutorial' / '15_download_progress.md' +PLUGIN_FILE = PROJECT_DIR / 'src' / 'jmcomic' / 'jm_plugin.py' +DOWNLOADER_FILE = PROJECT_DIR / 'src' / 'jmcomic' / 'jm_downloader.py' +RICH_INSTALLED = importlib.util.find_spec('rich') is not None + + +def create_album(episode_list=None): + if episode_list is None: + episode_list = [('101', '1', 'chapter-1'), ('102', '2', 'chapter-2')] + return JmAlbumDetail( + album_id='123456', + scramble_id='220980', + name='album', + episode_list=episode_list, + page_count=5, + pub_date='', + update_date='', + likes='0', + views='0', + comment_count=0, + works=[], + actors=[], + authors=['author'], + tags=['tag'], + ) + + +class FakeClient: + + image_counts = {'101': 2, '102': 3} + + def __init__(self, album): + self.album = album + + def get_album_detail(self, album_id): + assert str(album_id) == self.album.id + return self.album + + def check_photo(self, photo): + photo.page_arr = [ + f'{index:05}.jpg' + for index in range(1, self.image_counts[photo.id] + 1) + ] + photo.data_original_domain = 'cdn.example' + photo.data_original_query_params = 'v=1' + + +class FakeAsyncClient(FakeClient): + + async def setup(self): + pass + + async def close(self): + pass + + async def get_album_detail(self, album_id): + return super().get_album_detail(album_id) + + async def check_photo(self, photo): + super().check_photo(photo) + + +class Test_DownloadProgress(unittest.TestCase): + + def test_progress_display_id_adds_prefix_only_once(self): + self.assertEqual('JM123456', ProgressDownloader.display_id('123456')) + self.assertEqual('JM123456', ProgressDownloader.display_id('JM123456')) + + def test_downloader_use_logs_before_and_after_class(self): + class BeforeDownloader(JmDownloader): + pass + + class AfterDownloader(JmDownloader): + pass + + original_downloader = JmModuleConfig.CLASS_DOWNLOADER + JmModuleConfig.CLASS_DOWNLOADER = BeforeDownloader + try: + with self.assertLogs('jmcomic', level='INFO') as captured: + AfterDownloader.use() + + output = '\n'.join(captured.output) + self.assertEqual('downloader.use', captured.records[0].topic) + self.assertIn( + f'{BeforeDownloader.__module__}.{BeforeDownloader.__qualname__}', + output, + ) + self.assertIn( + f'{AfterDownloader.__module__}.{AfterDownloader.__qualname__}', + output, + ) + self.assertIs(AfterDownloader, JmModuleConfig.downloader_class()) + finally: + JmModuleConfig.CLASS_DOWNLOADER = original_downloader + + def test_async_downloader_use_logs_and_changes_async_default(self): + class BeforeAsyncDownloader(JmAsyncDownloader): + pass + + class AfterAsyncDownloader(JmAsyncDownloader): + pass + + original_downloader = JmModuleConfig.CLASS_ASYNC_DOWNLOADER + JmModuleConfig.CLASS_ASYNC_DOWNLOADER = BeforeAsyncDownloader + try: + with self.assertLogs('jmcomic', level='INFO') as captured: + AfterAsyncDownloader.use() + + output = '\n'.join(captured.output) + self.assertEqual('async_downloader.use', captured.records[0].topic) + self.assertIn( + f'{BeforeAsyncDownloader.__module__}.{BeforeAsyncDownloader.__qualname__}', + output, + ) + self.assertIn( + f'{AfterAsyncDownloader.__module__}.{AfterAsyncDownloader.__qualname__}', + output, + ) + self.assertIs( + AfterAsyncDownloader, + JmModuleConfig.async_downloader_class(), + ) + + with patch.object(AfterAsyncDownloader, '__init__', return_value=None): + downloader = new_async_downloader(option=object()) + self.assertIsInstance(downloader, AfterAsyncDownloader) + finally: + JmModuleConfig.CLASS_ASYNC_DOWNLOADER = original_downloader + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_sync_progress_is_rendered_before_download_finishes(self): + from rich.console import Console + + album = create_album() + photo = album[0] + FakeClient(album).check_photo(photo) + image = photo[0] + ui_output = StringIO() + original_console = ProgressDownloader.progress_console + ProgressDownloader.progress_console = Console( + file=ui_output, + force_terminal=True, + force_interactive=True, + color_system='standard', + width=100, + ) + downloader = None + + try: + option = create_option_by_str('{}') + with patch.object( + ProgressDownloader, + 'create_client', + return_value=FakeClient(album), + ): + downloader = ProgressDownloader(option) + downloader.before_album(album) + rendered_before_wait = ui_output.getvalue() + time.sleep(0.2) + self.assertEqual(rendered_before_wait, ui_output.getvalue()) + + downloader.before_photo(photo) + image.save_path = 'mock.jpg' + downloader.after_image(image, image.save_path) + + rendered_during_download = ui_output.getvalue() + self.assertIn('本子-JM123456', rendered_during_download) + self.assertIn('章节-JM101', rendered_during_download) + self.assertIn('1/2', rendered_during_download) + self.assertNotIn('✓ 本子-JM123456', rendered_during_download) + finally: + if downloader is not None: + downloader.stop_progress() + ProgressDownloader.progress_console = original_console + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_default_progress_console_respects_non_interactive_output(self): + original_console = ProgressDownloader.progress_console + ProgressDownloader.progress_console = None + try: + with patch('sys.stdout', StringIO()): + console = ProgressDownloader.get_progress_console() + self.assertFalse(console.is_terminal) + self.assertFalse(console.is_interactive) + finally: + ProgressDownloader.progress_console = original_console + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_non_interactive_notice_has_stable_text_layout(self): + from rich.console import Console + + ui_output = StringIO() + console = Console( + file=ui_output, + force_terminal=False, + force_interactive=False, + width=80, + ) + log_path = Path('downloads') / DownloadProgressPlugin.log_file + DownloadProgressPlugin.print_non_interactive_notice( + console, + log_path, + ) + + rendered = ui_output.getvalue() + lines = rendered.splitlines() + self.assertIn('JMComic Progress', lines[0]) + self.assertTrue(any('✓ 下载进度插件已启用' in line for line in lines)) + self.assertTrue(any('显示模式：完成后汇总' in line for line in lines)) + self.assertTrue(any( + '动态进度：请在 Terminal / PowerShell 中运行' in line + for line in lines + )) + self.assertTrue(any(line.strip(' │') == '详细日志' for line in lines)) + self.assertTrue(any( + str(log_path) in line + for line in lines + )) + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_log_panel_keeps_latest_six_lines(self): + from rich.console import Console + + ui_output = StringIO() + console = Console( + file=ui_output, + force_terminal=False, + force_interactive=False, + width=100, + ) + ProgressDownloader.reset_progress_logs() + try: + for index in range(8): + ProgressDownloader.append_progress_log(f'log-{index}') + console.print(ProgressDownloader.build_log_panel()) + finally: + ProgressDownloader.reset_progress_logs() + + rendered = ui_output.getvalue() + self.assertIn('JMComic Logs', rendered) + self.assertNotIn('log-0', rendered) + self.assertNotIn('log-1', rendered) + for index in range(2, 8): + self.assertIn(f'log-{index}', rendered) + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_non_interactive_progress_only_prints_final_summary(self): + from rich.console import Console + + album = create_album([('101', '1', 'chapter-1')]) + photo = album[0] + FakeClient(album).check_photo(photo) + image = photo[0] + ui_output = StringIO() + original_console = ProgressDownloader.progress_console + ProgressDownloader.progress_console = Console( + file=ui_output, + force_terminal=False, + force_interactive=False, + width=100, + ) + downloader = None + + try: + option = create_option_by_str('{}') + with patch.object( + ProgressDownloader, + 'create_client', + return_value=FakeClient(album), + ): + downloader = ProgressDownloader(option) + downloader.before_album(album) + downloader.before_photo(photo) + self.assertEqual('', ui_output.getvalue()) + + image.save_path = 'mock.jpg' + downloader.after_image(image, image.save_path) + self.assertEqual('', ui_output.getvalue()) + + second_image = photo[1] + second_image.save_path = 'mock-2.jpg' + downloader.after_image(second_image, second_image.save_path) + downloader.after_photo(photo) + self.assertEqual('', ui_output.getvalue()) + + downloader.after_album(album) + self.assertEqual( + '✓ 下载完成：本子-JM123456，章节 1/1，图片 2/2\n', + ui_output.getvalue(), + ) + finally: + if downloader is not None: + downloader.stop_progress() + ProgressDownloader.progress_console = original_console + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_plugin_does_not_close_external_log_handlers(self): + original_handlers = jm_logger.handlers[:] + external_handler = logging.Handler() + jm_logger.handlers[:] = [external_handler] + + try: + with TemporaryDirectory() as temp_dir: + previous_cwd = os.getcwd() + os.chdir(temp_dir) + try: + with patch.object( + external_handler, + 'close', + wraps=external_handler.close, + ) as close_handler: + DownloadProgressPlugin(object()).redirect_log_to_file() + self.assertNotIn(external_handler, jm_logger.handlers) + close_handler.assert_not_called() + finally: + for handler in jm_logger.handlers[:]: + jm_logger.removeHandler(handler) + handler.close() + os.chdir(previous_cwd) + finally: + jm_logger.handlers[:] = original_handlers + external_handler.close() + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_plugin_uses_real_scheduler_without_stdout_or_image_io(self): + from rich.console import Console + + album = create_album() + ui_output = StringIO() + stdout_output = StringIO() + original_console = ProgressDownloader.progress_console + original_downloader = JmModuleConfig.CLASS_DOWNLOADER + original_async_downloader = JmModuleConfig.CLASS_ASYNC_DOWNLOADER + ProgressDownloader.progress_console = Console( + file=ui_output, + force_terminal=True, + force_interactive=True, + color_system='standard', + width=100, + ) + + def fake_create_client(_downloader): + return FakeClient(album) + + def fake_download_image(downloader, image): + image.save_path = str(Path.cwd() / image.filename) + downloader.before_image(image, image.save_path) + downloader.after_image(image, image.save_path) + + try: + with TemporaryDirectory() as temp_dir: + previous_cwd = os.getcwd() + os.chdir(temp_dir) + try: + with patch.object(ProgressDownloader, 'create_client', fake_create_client), \ + patch.object(ProgressDownloader, 'download_by_image_detail', fake_download_image), \ + contextlib.redirect_stdout(stdout_output): + with jm_task_context(cli_no_progress=True): + option = create_option_by_str(''' +plugins: + after_init: + - plugin: download_progress +''') + download_album('123456', option) + + for handler in jm_logger.handlers: + handler.flush() + + log_text = Path('jmcomic-download.log').read_text(encoding='utf-8') + self.assertIn('已将默认 Downloader 替换为 ProgressDownloader', log_text) + self.assertIn('album.before', log_text) + self.assertIn('image.after', log_text) + self.assertIn('album.after', log_text) + self.assertNotIn(' INFO:', log_text) + self.assertEqual( + 1, + sum( + isinstance(handler, logging.FileHandler) + for handler in jm_logger.handlers + ), + ) + self.assertEqual(2, len(jm_logger.handlers)) + finally: + for handler in jm_logger.handlers[:]: + jm_logger.removeHandler(handler) + handler.close() + os.chdir(previous_cwd) + finally: + ProgressDownloader.progress_console = original_console + JmModuleConfig.CLASS_DOWNLOADER = original_downloader + JmModuleConfig.CLASS_ASYNC_DOWNLOADER = original_async_downloader + setup_default_jm_logger() + + rendered = ui_output.getvalue() + self.assertEqual('', stdout_output.getvalue()) + self.assertIn('\x1b[', rendered) + self.assertIn('彩色下载进度插件已启用', rendered) + self.assertIn('检测到命令行参数 --no-progress', rendered) + self.assertIn('当前 Option 已配置 download_progress', rendered) + self.assertIn('JMComic Logs', rendered) + self.assertIn('album.before', rendered) + self.assertIn('章节-JM101', rendered) + self.assertIn('章节-JM102', rendered) + self.assertIn('✓ 本子-JM123456', rendered) + + @unittest.skipUnless(RICH_INSTALLED, '需要安装 rich 才能测试彩色进度插件') + def test_plugin_supports_real_async_scheduler_without_network_or_image_io(self): + from rich.console import Console + + album = create_album() + ui_output = StringIO() + stdout_output = StringIO() + original_console = ProgressDownloader.progress_console + original_downloader = JmModuleConfig.CLASS_DOWNLOADER + original_async_downloader = JmModuleConfig.CLASS_ASYNC_DOWNLOADER + ProgressDownloader.progress_console = Console( + file=ui_output, + force_terminal=False, + force_interactive=False, + width=100, + ) + + async def fake_download_image(downloader, image): + image.save_path = str(Path.cwd() / image.filename) + await downloader.before_image(image, image.save_path) + await downloader.after_image(image, image.save_path) + + try: + with TemporaryDirectory() as temp_dir: + previous_cwd = os.getcwd() + os.chdir(temp_dir) + try: + with contextlib.redirect_stdout(stdout_output): + option = create_option_by_str(''' +plugins: + after_init: + - plugin: download_progress +''') + with patch.object( + option, + 'new_jm_async_client', + return_value=FakeAsyncClient(album), + ), patch.object( + AsyncProgressDownloader, + '_download_single_image', + fake_download_image, + ): + asyncio.run(download_album_async('123456', option)) + + for handler in jm_logger.handlers: + handler.flush() + + log_text = Path('jmcomic-download.log').read_text(encoding='utf-8') + self.assertIn( + '已将默认 Async Downloader 替换为 AsyncProgressDownloader', + log_text, + ) + self.assertIn('album.before', log_text) + self.assertIn('image.after', log_text) + self.assertIn('album.after', log_text) + finally: + for handler in jm_logger.handlers[:]: + jm_logger.removeHandler(handler) + handler.close() + os.chdir(previous_cwd) + finally: + ProgressDownloader.progress_console = original_console + JmModuleConfig.CLASS_DOWNLOADER = original_downloader + JmModuleConfig.CLASS_ASYNC_DOWNLOADER = original_async_downloader + setup_default_jm_logger() + + rendered = ui_output.getvalue() + self.assertEqual('', stdout_output.getvalue()) + self.assertIn('显示模式：完成后汇总', rendered) + self.assertIn('动态进度：请在 Terminal / PowerShell 中运行', rendered) + self.assertIn('详细日志', rendered) + self.assertIn('✓ 下载完成：本子-JM123456，章节 2/2，图片 5/5', rendered) + + def test_plugin_is_registered_and_documentation_only_shows_usage(self): + document = DOCUMENT_FILE.read_text(encoding='utf-8') + self.assertIs( + DownloadProgressPlugin, + JmModuleConfig.REGISTRY_PLUGIN['download_progress'], + ) + self.assertIn('plugin: download_progress', document) + self.assertIn('download_album_async', document) + self.assertNotIn('class ProgressDownloader', document) + self.assertNotIn('class DownloadProgressPlugin', document) + self.assertIn('class ProgressDownloader', PLUGIN_FILE.read_text(encoding='utf-8')) + self.assertNotIn('class ProgressDownloader', DOWNLOADER_FILE.read_text(encoding='utf-8')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_jmcomic/test_jm_feature.py b/tests/test_jmcomic/test_jm_feature.py index 1a65743c6..8106faaea 100644 --- a/tests/test_jmcomic/test_jm_feature.py +++ b/tests/test_jmcomic/test_jm_feature.py @@ -48,42 +48,43 @@ def invoke(self, option, **kwargs): pass base = MyFeature() - self.assertTrue(base.should_invoke('download_album', 'after_album')) - self.assertTrue(base.should_invoke('download_album', 'after_photo')) + self.assertTrue(base.should_invoke('after_album')) + self.assertTrue(base.should_invoke('after_photo')) - # PluginFeature 根据来源推导执行时机 + # PluginFeature 根据当前 TaskContext 推导执行时机 pf = Feature.export_pdf - # download_album → 只在 after_album 执行 - self.assertTrue(pf.should_invoke('download_album', 'after_album')) - self.assertFalse(pf.should_invoke('download_album', 'after_photo')) - # download_photo → 只在 after_photo 执行 - self.assertTrue(pf.should_invoke('download_photo', 'after_photo')) - self.assertFalse(pf.should_invoke('download_photo', 'after_album')) + with jm_task_context(download_type='album'): + self.assertTrue(pf.should_invoke('after_album')) + self.assertFalse(pf.should_invoke('after_photo')) + + with jm_task_context(download_type='photo'): + self.assertTrue(pf.should_invoke('after_photo')) + self.assertFalse(pf.should_invoke('after_album')) def test_adapt_kwargs(self): """测试 PluginFeature 参数动态适配""" when = 'after_album' pdf = Feature.export_pdf - adapted = pdf._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = pdf._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['filename_rule'], '[JM{Aid}]{Atitle}') zip_f = Feature.export_zip - adapted = zip_f._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = zip_f._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['filename_rule'], '[JM{Aid}]{Atitle}') long_img = Feature.export_long_img - adapted = long_img._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = long_img._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['filename_rule'], '[JM{Aid}]{Atitle}') - # download_photo 模式 + # after_photo 使用章节命名规则 when = 'after_photo' - adapted = pdf._adapt_plugin_kwargs(self.option, 'download_photo', when) + adapted = pdf._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['filename_rule'], '[JM{Pid}]{Ptitle}') # 用户显式传入的参数不被动态适配 (通过 kwargs 机制自带) custom = Feature.export_zip(filename_rule='Ptitle') - adapted = custom._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = custom._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['filename_rule'], 'Ptitle') # 用户显式指定，不被 setdefault 覆盖 def test_dynamic_base_dir(self): @@ -92,20 +93,20 @@ def test_dynamic_base_dir(self): when = 'after_album' # 1. PDF - adapted = Feature.export_pdf._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = Feature.export_pdf._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['pdf_dir'], './custom_base') # 2. ZIP - adapted = Feature.export_zip._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = Feature.export_zip._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['zip_dir'], './custom_base') # 3. LongImg - adapted = Feature.export_long_img._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = Feature.export_long_img._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['img_dir'], './custom_base') # 4. 如果显式指定了，则不应被覆盖 custom_pdf = Feature.export_pdf(pdf_dir='./explicit_dir') - adapted = custom_pdf._adapt_plugin_kwargs(self.option, 'download_album', when) + adapted = custom_pdf._adapt_plugin_kwargs(self.option, when) self.assertEqual(adapted['pdf_dir'], './explicit_dir') def test_download_use_feature(self): @@ -197,4 +198,4 @@ def test_export_album_use_photo_rule(self): # 因为在 download_album 的 after_album 阶段，photo 为 None with self.assertRaises(AttributeError): album = self.client.get_album_detail(album_id) - f.invoke(self.option, feature_from='download_album', when='after_album', album=album, photo=None) + f.invoke(self.option, when='after_album', album=album, photo=None) diff --git a/tests/test_jmcomic/test_jm_release.py b/tests/test_jmcomic/test_jm_release.py new file mode 100644 index 000000000..500f8d522 --- /dev/null +++ b/tests/test_jmcomic/test_jm_release.py @@ -0,0 +1,159 @@ +"""Tests for changelog-driven GitHub Release metadata.""" + +import importlib.util +import os +import tempfile +from pathlib import Path +from unittest.mock import patch + +from test_jmcomic import * + + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +RELEASE_SCRIPT = PROJECT_ROOT / ".github" / "release.py" +SPEC = importlib.util.spec_from_file_location("jmcomic_release", RELEASE_SCRIPT) +if SPEC is None or SPEC.loader is None: + raise RuntimeError(f"Unable to load release script: {RELEASE_SCRIPT}") +release = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(release) + + +class Test_Release(unittest.TestCase): + + def create_project(self, root: Path, version: str, changelog: str) -> None: + package_dir = root / "src" / "jmcomic" + package_dir.mkdir(parents=True) + (package_dir / "__init__.py").write_text(f"__version__ = '{version}'\n", encoding="utf-8") + (root / "CHANGELOG.md").write_text(changelog, encoding="utf-8") + + def test_release_body_comes_from_matching_changelog_section(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self.create_project( + root, + "2.7.4", + """# Changelog + +## [2.7.4] - 2026-08-08 + +### Added +- Manifest release notes. + +## [2.7.3] - 2026-07-01 + +### Fixed +- Previous release. +""", + ) + + tag, body = release.build_release_metadata("v2.7.4: text ignored by release body", root) + + self.assertEqual(tag, "v2.7.4") + self.assertEqual(body, "### Added\n- Manifest release notes.") + + def test_rejects_commit_and_source_version_mismatch(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self.create_project( + root, + "2.7.4", + "## [2.7.4] - 2026-08-08\n\n### Added\n- Entry.\n", + ) + + with self.assertRaisesRegex(ValueError, "Version mismatch"): + release.build_release_metadata("v2.7.5: wrong version", root) + + def test_manual_release_uses_source_version_without_commit_message(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self.create_project( + root, + "2.7.4", + "## [2.7.4] - 2026-08-08\n\n### Fixed\n- Manual recovery.\n", + ) + + tag, body = release.build_release_metadata(root_dir=root) + + self.assertEqual(tag, "v2.7.4") + self.assertEqual(body, "### Fixed\n- Manual recovery.") + + def test_rejects_missing_duplicate_or_empty_changelog_section(self): + cases = ( + "## [2.7.3] - 2026-08-01\n\n### Fixed\n- Old.\n", + "## [2.7.4] - 2026-08-08\n- First.\n\n## [2.7.4] - 2026-08-07\n- Duplicate.\n", + "## [2.7.4] - 2026-08-08\n\n## [2.7.3] - 2026-08-01\n- Old.\n", + ) + for changelog in cases: + with self.subTest(changelog=changelog), tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + self.create_project(root, "2.7.4", changelog) + with self.assertRaises(ValueError): + release.build_release_metadata("v2.7.4: release", root) + + def test_main_writes_changelog_body_and_tag_output(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + output_path = root / "github-output.txt" + self.create_project( + root, + "2.7.4", + "## [2.7.4] - 2026-08-08\n\n### Fixed\n- Reliable release notes.\n", + ) + + with patch.object(release, "ROOT_DIR", root), patch.dict(os.environ, {"GITHUB_OUTPUT": str(output_path)}): + self.assertEqual(release.main("v2.7.4: commit summary only"), 0) + + self.assertEqual( + (root / "release_body.txt").read_text(encoding="utf-8"), + "### Fixed\n- Reliable release notes.\n", + ) + self.assertEqual(output_path.read_text(encoding="utf-8"), "tag=v2.7.4\n") + + def test_release_entry_count_uses_changelog_bullets(self): + body = "### Added\n- First.\n- Second.\n\n### Fixed\n- Third." + + self.assertEqual(release.count_release_entries(body), 3) + + def test_workflow_keeps_master_v_prefix_trigger_without_generated_notes(self): + workflow = (PROJECT_ROOT / ".github" / "workflows" / "release_auto.yml").read_text(encoding="utf-8") + + self.assertIn("branches:\n - master", workflow) + self.assertIn("startsWith(github.event.head_commit.message, 'v')", workflow) + self.assertIn('python .github/release.py "$commit_message"', workflow) + self.assertNotIn("generate_release_notes:", workflow) + + def test_manual_workflow_reads_source_version_from_master(self): + workflow = (PROJECT_ROOT / ".github" / "workflows" / "release.yml").read_text(encoding="utf-8") + + self.assertIn("workflow_dispatch:", workflow) + self.assertNotIn("types: [ published ]", workflow) + self.assertIn("github.ref_name == 'master'", workflow) + self.assertIn("python .github/release.py\n", workflow) + self.assertIn("softprops/action-gh-release@v2", workflow) + self.assertIn("pypa/gh-action-pypi-publish@release/v1", workflow) + + def test_release_workflows_build_before_creating_release(self): + for filename in ("release.yml", "release_auto.yml"): + with self.subTest(filename=filename): + workflow = (PROJECT_ROOT / ".github" / "workflows" / filename).read_text(encoding="utf-8") + + self.assertLess(workflow.index("- name: Build\n"), workflow.index("- name: Create Release\n")) + + def test_test_workflows_watch_development_requirements(self): + for filename in ("test_api.yml", "test_html.yml"): + with self.subTest(filename=filename): + workflow = (PROJECT_ROOT / ".github" / "workflows" / filename).read_text(encoding="utf-8") + + self.assertIn(" - '.github/requirements-dev.txt'", workflow) + + def test_contributing_allows_only_formal_release_prs_to_master(self): + contributing = (PROJECT_ROOT / ".github" / "CONTRIBUTING.md").read_text(encoding="utf-8") + + self.assertIn("普通 PR 禁止直飞 master", contributing) + self.assertIn("发版专线 (仅限版本发布)", contributing) + self.assertIn("任意一项缺失，都不得指向或合并到 `master`", contributing) + self.assertNotIn("本项目不接受任何直接指向 `master` 分支的 PR", contributing) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_jmcomic/test_jm_task_context.py b/tests/test_jmcomic/test_jm_task_context.py index e052640f5..89d585958 100644 --- a/tests/test_jmcomic/test_jm_task_context.py +++ b/tests/test_jmcomic/test_jm_task_context.py @@ -4,9 +4,11 @@ import unittest from concurrent.futures import ThreadPoolExecutor from queue import Queue +from unittest.mock import patch from jmcomic import ( BaseDownloader, + DownloadManifest, Feature, JmAsyncDownloader, JmDownloader, @@ -166,31 +168,39 @@ def make_record(topic, context, level=logging.INFO): ).startswith(formatter.WARN_COLOR)) def test_public_downloads_add_task_context_to_downloader_logs(self): - class FakeSyncDownloader: + class AlbumDetail: + duration = None - def __init__(self, _option): - pass + @staticmethod + def is_album(): + return True - def __enter__(self): - return self + class PhotoDetail: + duration = None + from_album = object() - def __exit__(self, *_args): - pass + @staticmethod + def is_album(): + return False - def add_features(self, *_args): - pass + class FakeSyncDownloader(BaseDownloader): - def download_album(self, album_id): - jm_log('album.before', 'message') - return album_id + def __enter__(self): + return self - def raise_if_has_exception(self): + def __exit__(self, *_args): pass - class FakeAsyncDownloader: + def download_album(self, _album_id): + detail = AlbumDetail() + self.begin_manifest(detail) + try: + jm_log('album.before', 'message') + finally: + self.finish_manifest(detail) + return detail - def __init__(self, _option): - pass + class FakeAsyncDownloader(BaseDownloader): async def __aenter__(self): return self @@ -198,15 +208,14 @@ async def __aenter__(self): async def __aexit__(self, *_args): pass - def add_features(self, *_args): - pass - - async def download_photo(self, photo_id): - jm_log('photo.before', 'message') - return photo_id - - def raise_if_has_exception(self): - pass + async def download_photo(self, _photo_id): + detail = PhotoDetail() + self.begin_manifest(detail) + try: + jm_log('photo.before', 'message') + finally: + self.finish_manifest(detail) + return detail handler = ListHandler() original_handlers = jm_logger.handlers[:] @@ -236,6 +245,8 @@ def raise_if_has_exception(self): ['album.before', 'photo.before'], [record.topic for record in task_records], ) + album_context = task_records[0].jm_task_context + photo_context = task_records[1].jm_task_context self.assertEqual( { 'session_id': 'session-1', @@ -243,7 +254,7 @@ def raise_if_has_exception(self): 'download_type': 'album', 'jm_id': '123', }, - task_records[0].jm_task_context, + {key: album_context[key] for key in ('session_id', 'task_id', 'download_type', 'jm_id')}, ) self.assertEqual( { @@ -252,8 +263,88 @@ def raise_if_has_exception(self): 'download_type': 'photo', 'jm_id': '456', }, - task_records[1].jm_task_context, + {key: photo_context[key] for key in ('session_id', 'task_id', 'download_type', 'jm_id')}, ) + self.assertIsInstance(album_context.get('task_started_at'), float) + self.assertIsInstance(photo_context.get('task_started_at'), float) + + def test_sync_result_duration_uses_task_context_and_finishes_after_downloader_exit(self): + clock = {'now': 10.0} + observed_contexts = [] + + class Detail: + duration = 99.0 + + @staticmethod + def is_album(): + return True + + class FakeDownloader(BaseDownloader): + + def __enter__(self): + return self + + def __exit__(self, *_args): + clock['now'] = 20.0 + + def download_album(self, _album_id): + detail = Detail() + self.begin_manifest(detail) + try: + observed_contexts.append(get_jm_task_context()) + finally: + self.finish_manifest(detail) + return detail + + with patch('jmcomic.api.perf_counter', side_effect=lambda: clock['now'], create=True): + result = download_album('123', option=object(), downloader=FakeDownloader) + + self.assertEqual(10.0, observed_contexts[0].get('task_started_at')) + self.assertEqual(10.0, result.manifest.duration) + self.assertEqual(10.0, result.duration) + self.assertEqual(99.0, result.detail.duration) + self.assertEqual({}, get_jm_task_context()) + + def test_async_result_duration_uses_task_context_and_finishes_after_downloader_exit(self): + async def run_test(): + clock = {'now': 100.0} + observed_contexts = [] + + class Detail: + duration = 99.0 + from_album = object() + + @staticmethod + def is_album(): + return False + + class FakeDownloader(BaseDownloader): + + async def __aenter__(self): + observed_contexts.append(get_jm_task_context()) + return self + + async def __aexit__(self, *_args): + clock['now'] = 130.0 + + async def download_photo(self, _photo_id): + detail = Detail() + self.begin_manifest(detail) + self.finish_manifest(detail) + return detail + + with patch('jmcomic.api.perf_counter', side_effect=lambda: clock['now'], create=True): + result = await download_photo_async('456', option=object(), downloader=FakeDownloader) + + return result, observed_contexts + + result, observed_contexts = asyncio.run(run_test()) + + self.assertEqual(100.0, observed_contexts[0].get('task_started_at')) + self.assertEqual(30.0, result.manifest.duration) + self.assertEqual(30.0, result.duration) + self.assertEqual(99.0, result.detail.duration) + self.assertEqual({}, get_jm_task_context()) def test_plugin_invocation_can_read_current_task_context(self): observed = [] @@ -299,24 +390,54 @@ def test_feature_invocation_can_read_current_task_context(self): class TaskAwareFeature(Feature): - def invoke(self, _option, feature_from, when, **_kwargs): - observed.append((feature_from, when, self.jm_task_context)) + def invoke(self, _option, when, **_kwargs): + observed.append((when, self.jm_task_context)) downloader = BaseDownloader(object()) - downloader.add_features(TaskAwareFeature(), 'download_album') - with jm_task_context(session_id='feature-session', task_id='feature-task'): + with jm_task_context( + session_id='feature-session', + task_id='feature-task', + download_type='album', + ): + downloader.add_features(TaskAwareFeature()) downloader._invoke_features_for('after_album') self.assertEqual([( - 'download_album', 'after_album', { 'session_id': 'feature-session', 'task_id': 'feature-task', + 'download_type': 'album', }, )], observed) + def test_add_features_requires_download_type_context(self): + downloader = BaseDownloader(object()) + + with self.assertRaisesRegex(Exception, 'Feature 注册与执行必须位于下载任务上下文中'): + downloader.add_features(Feature()) + + def test_feature_invocation_requires_download_type_context(self): + downloader = BaseDownloader(object()) + + with jm_task_context(download_type='album'): + downloader.add_features(Feature()) + + with self.assertRaisesRegex(Exception, 'Feature 注册与执行必须位于下载任务上下文中'): + downloader._invoke_features_for('after_album') + + def test_plugin_feature_uses_download_type_context(self): + feature = Feature.export_pdf + + with jm_task_context(download_type='album'): + self.assertTrue(feature.should_invoke('after_album')) + self.assertFalse(feature.should_invoke('after_photo')) + + with jm_task_context(download_type='photo'): + self.assertTrue(feature.should_invoke('after_photo')) + self.assertFalse(feature.should_invoke('after_album')) + def test_sync_batch_binds_parent_and_item_context(self): def fake_download(jmid, _option, _downloader, **_kwargs): context = get_jm_task_context() @@ -383,14 +504,19 @@ def fail(jmid, _option, _downloader, **_kwargs): jm_logger.handlers[:] = original_handlers self.assertIn('404', result.failed) - self.assertEqual(1, len(handler.records)) + batch_failure_records = [ + record + for record in handler.records + if getattr(record, 'topic', None) == 'batch.failed' + ] + self.assertEqual(1, len(batch_failure_records)) self.assertEqual( { 'session_id': 'failed-session', 'download_type': 'fail', 'jm_id': '404', }, - handler.records[0].jm_task_context, + batch_failure_records[0].jm_task_context, ) def test_sync_downloader_propagates_both_threading_branches(self):