diff --git a/README.md b/README.md index 5fbb5711..eebf0bbc 100644 --- a/README.md +++ b/README.md @@ -53,9 +53,10 @@ - 登录 - 搜索本子（支持所有搜索项） -- 图片下载解码 +- 获取本子评论（包括回评、剧透标识） - 分类/排行榜 -- 本子/章节详情 +- 获取本子/章节详情 +- 图片下载解码 - 个人收藏夹 - 接口加解密（APP的接口） @@ -221,15 +222,16 @@ jmv 350234 -y - **绕过Cloudflare的反爬虫** - **实现禁漫APP接口最新的加解密算法 (1.6.3)** +- 支持**Async**和**Sync**两套 API - 用法多样： - GitHub Actions：网页上直接输入本子id就能下载（[教程：使用GitHub Actions下载禁漫本子](./assets/docs/sources/tutorial/1_github_actions.md)） - 命令行：无需写Python代码，简单易用（[教程：使用命令行下载禁漫本子](./assets/docs/sources/tutorial/2_command_line.md)） - Python代码：最本质、最强大的使用方式，需要你有一定的python编程基础 -- **支持 Async 和 Sync 两套 API** - 支持**网页端**和**移动端**两种客户端实现，可通过配置切换（**移动端不限ip兼容性好，网页端限制ip地区但效率高**） - 支持**自动重试和域名切换**机制 +- 支持按下载任务维度结构化收集日志 - **可配置性强** - 不配置也能使用，十分方便 @@ -238,7 +240,7 @@ jmv 350234 -y 等 - **可扩展性强** - - 支持自定义本子/章节/图片下载前后的回调函数 + - 支持自定义本子/章节/图片下载前后事件的回调函数 - 支持自定义类：`Downloader（负责调度）` `Option（负责配置）` `Client（负责请求）` `实体类`等 - 支持自定义日志、异常监听器 - **支持Plugin插件，可以方便地扩展功能，以及使用别人的插件，目前核心内置插件有**： diff --git a/assets/docs/sources/api/command-line.md b/assets/docs/sources/api/command-line.md index 6f44be8e..5e2d9304 100644 --- a/assets/docs/sources/api/command-line.md +++ b/assets/docs/sources/api/command-line.md @@ -1,6 +1,6 @@ # command-line -::: jmcomic.cl +::: jmcomic.cli options: members: - JmcomicUI diff --git a/assets/docs/sources/api/entity.md b/assets/docs/sources/api/entity.md index a019598c..379a0e65 100644 --- a/assets/docs/sources/api/entity.md +++ b/assets/docs/sources/api/entity.md @@ -8,4 +8,6 @@ - JmPhotoDetail - JmImageDetail - JmPageContent - - JmSearchPage \ No newline at end of file + - JmAlbumComment + - JmAlbumCommentPage + - JmSearchPage diff --git a/assets/docs/sources/index.md b/assets/docs/sources/index.md index 89d4ce8e..8d06270f 100644 --- a/assets/docs/sources/index.md +++ b/assets/docs/sources/index.md @@ -65,11 +65,11 @@ - + - - - Repo Card + + + Repo Card @@ -83,4 +83,4 @@ - \ No newline at end of file + diff --git a/assets/docs/sources/option_file_syntax.md b/assets/docs/sources/option_file_syntax.md index e676e94f..49e0975f 100644 --- a/assets/docs/sources/option_file_syntax.md +++ b/assets/docs/sources/option_file_syntax.md @@ -186,8 +186,8 @@ plugins: # v2.5.0 引入的插件 # 可以启动一个服务器，可以在浏览器上查看本子 - # 基于flask框架，需要安装额外库: [pip install plugin_jm_server] - # 源码：https://github.com/hect0x7/plugin-jm-server + # 基于flask框架，需要安装额外库: [pip install jm-view-server] + # 源码：https://github.com/hect0x7/jm-view-server - plugin: jm_server kwargs: password: '3333' # 服务器访问密码 @@ -227,10 +227,10 @@ plugins: rule: '{Atitle}/{Aid}_cover.jpg' - after_album: # 钩子（插件被调用时机） + after_album: # 事件（插件被调用的时机） - plugin: zip # 压缩文件插件 kwargs: - # 压缩文件插件，配在不同钩子下面，效果不一样。可以选择配在 after_album 或者 after_photo 下 + # 压缩文件插件，配在不同事件下面，效果不一样。可以选择配在 after_album 或者 after_photo 下 # 配置在 after_album 下 → 整个本子合并为一个压缩文件 # 配置在 after_photo 下 → 每个章节各一个压缩文件 # （旧的 level 配置已废弃，如果你配置过level，比如level=photo, 请直接改用after_photo） @@ -239,7 +239,7 @@ plugins: suffix: zip #压缩包后缀名，默认值为zip，可以指定为zip或者7z filename_rule: Atitle # 压缩文件的命名规则 # 请注意⚠ [https://github.com/hect0x7/JMComic-Crawler-Python/issues/223#issuecomment-2045227527] - # filename_rule和所在钩子有对应关系 + # filename_rule和所在事件有对应关系 # 如果配置在 after_photo 下, filename_rule 可以写 Pxxx 和Axxx # 如果配置在 after_album 下, filename_rule 只能写 Axxx，不能写 Pxxx diff --git a/assets/docs/sources/tutorial/0_common_usage.md b/assets/docs/sources/tutorial/0_common_usage.md index d6268a41..51e31a3d 100644 --- a/assets/docs/sources/tutorial/0_common_usage.md +++ b/assets/docs/sources/tutorial/0_common_usage.md @@ -109,7 +109,7 @@ except JmcomicException as e: # 而JmDownloader有对应字段记录了这些线程发生的异常 # 使用check_exception=True参数可以使downloader主动检查是否存在下载异常 # 如果有，则当前线程会主动上抛一个PartialDownloadFailedException异常 -# 该参数主要用于主动检查部分下载失败的情况， +# 该参数主要用于主动检查部分下载失败的情况，（仅对单个本子/章节 ID 生效，传入多个 ID 时不生效。多个 ID 的场景见下） # 因为非当前线程抛出的异常（比如下载章节的线程和下载图片的线程），这些线程如果抛出异常， # 当前线程是感知不到的，try-catch下载方法download_album不能捕获到其他线程发生的异常。 try: @@ -117,6 +117,12 @@ try: except PartialDownloadFailedException as e: downloader: JmDownloader = e.downloader print(f'下载出现部分失败, 下载失败的章节: {downloader.download_failed_photo}, 下载失败的图片: {downloader.download_failed_image}') + +# 多 ID 下载不会因为某一项失败而中断，请检查 BatchResult.failed。 +# 如果需要在批量失败时抛异常、重试或拿到更详细信息，建议自行封装一个批量下载方法。 +result = download_album([123, 456, 789]) +for album_id, error in result.failed.items(): + print(f'本子 {album_id} 下载失败: {error}') ``` @@ -164,10 +170,73 @@ for aid, atitle, tag_list in page.iter_id_title_tag(): # 使用page的iter_id_t download_album(aid_list, option) ``` +## 获取本子评论 + +```python +from jmcomic import JmOption, JmAlbumCommentPage, JmAlbumComment + +client = JmOption.default().new_jm_client(impl='api') + +# 获取第一页评论 +page: JmAlbumCommentPage = client.album_pagination('123456') + +print('本页评论数:', len(page)) +print('本页评论数（含回评）:', page.comment_count) +print('本子的评论总数:', page.total) +print('总页数:', page.page_count) + +# page对象可以直接遍历评论，评论类型是JmAlbumComment +comment: JmAlbumComment +for comment in page: + print('用户:', comment.nickname or comment.username) + print('内容:', comment.content) + print('是否剧透:', comment.is_spoiler) + + # 回评也是评论对象 + for reply in comment.replies: + print('回评用户:', reply.nickname or reply.username) + print('回评内容:', reply.content) + print('回评是否剧透:', reply.is_spoiler) + +# gen 方法支持自动循环获取评论分页，直到结束 +for page in client.album_pagination_gen('123456'): + print('本页主评论数:', len(page)) + print('本页评论数（含回评）:', page.comment_count) + print('主评论总数:', page.total) + print('总页数:', page.page_count) + + for comment in page: + print('用户:', comment.nickname or comment.username) + print('内容:', comment.content) + print('是否剧透:', comment.is_spoiler) +``` + ## 获取收藏夹 可参考discussions: https://github.com/hect0x7/JMComic-Crawler-Python/discussions/235 +### 一键导出全部收藏夹 + +下面的代码不会下载图片，只会把帐号中的全部收藏夹导出为 CSV，并生成 `favorites.zip`： + +```python +from jmcomic import JmOption, FavoriteFolderExportPlugin + +USERNAME = '你的禁漫帐号' +PASSWORD = '你的禁漫密码' + +option = JmOption.default() +option.build_jm_client().login(USERNAME, PASSWORD) + +FavoriteFolderExportPlugin(option).invoke( + save_dir='./', + zip_enable=True, + zip_filepath='./favorites.zip', +) +``` + +### 获取并遍历收藏夹 + ```python from jmcomic import * diff --git a/assets/docs/sources/tutorial/11_log_custom.md b/assets/docs/sources/tutorial/11_log_custom.md index 66aefb5d..9b6e893a 100644 --- a/assets/docs/sources/tutorial/11_log_custom.md +++ b/assets/docs/sources/tutorial/11_log_custom.md @@ -21,23 +21,45 @@ from jmcomic import disable_jm_log disable_jm_log() ``` -## 2. 日志过滤，只保留特定topic +## 2. 日志过滤，只保留特定 topic -使用插件配置 +最简单的方式是使用内置插件配置： ```yaml log: true plugins: after_init: - - plugin: log_topic_filter # 日志topic过滤插件 + - plugin: log_topic_filter # 日志 topic 过滤插件 kwargs: - whitelist: [ # 只保留api和html，这两个是Client发请求时会打的日志topic + whitelist: [ # 只保留 api 和 html，这两个是 Client 发请求时会打印的日志 topic 'api', 'html', ] ``` +这个插件底层使用标准 `logging.Filter`。jmcomic 将每条日志的 topic 放在 `LogRecord.topic` 中，需要通过代码自定义时可以直接使用 logging API： + +```python +import logging + +from jmcomic import jm_logger + + +class TopicFilter(logging.Filter): + def __init__(self, whitelist): + super().__init__() + self.whitelist = set(whitelist) + + def filter(self, record): + return getattr(record, 'topic', None) in self.whitelist + + +jm_logger.addFilter(TopicFilter({'api', 'html'})) +``` + +Filter 加在 `jm_logger` 上会作用于它的所有 Handler；如果只想过滤某个输出目标，也可以将 Filter 加到对应的 Handler 上。 + ## 3. 屏蔽插件的日志 给插件配置加上一个`log`配置项即可 @@ -52,7 +74,76 @@ plugins: whitelist: [ api, ] ``` -## 4. 深度自定义：两类不同的拦截手段 +## 4. 并发下载的日志上下文 + +场景：当你在同时运行多个下载任务时，你希望区分哪些日志属于哪些任务，或者你希望结构化展示下载任务 对应的 下载日志。 + +首先，你需要设定一个任务id，这一步是在定义你的**任务维度**。下载一个本子还是多个本子都可以算作一个任务。 + +```python +from jmcomic import download_album, jm_task_context + +with jm_task_context(task_id='task-1'): # 设置这个下载本子123的任务id为 task-1 + download_album(123) + +with jm_task_context(task_id='task-1'): # 设置这三个本子的任务id为 task-1 + download_album([123, 456, 789]) +``` + +设置了task_id以后，整个 download_album 内的日志打印都会具有这个task_id标识。 + +> jmcomic 使用 [ContextVar](https://docs.python.org/3/library/contextvars.html) 把上下文传播到库内部创建的下载线程或async task里。你设置的task_id其实就是上下文里的一个字段 + +最直接的效果是，默认终端日志会自动显示任务 ID： + +```text +[2026-01-01 12:00:00] [MainThread]:[task_id=task-1; album=123] 【album.after】本子下载完成: [123] +``` + +接下来你可以实现更高级的功能，比如收集任务对应的日志： + +给 `jm_logger` 添加一个 Handler。每当 jmcomic 产生一条日志，Handler 都可以从 `record.jm_task_context` 中取出它所属的 `task_id`，从而按任务收集日志。完整代码如下： + +```python +import logging +from collections import defaultdict + +from jmcomic import jm_logger, download_album, jm_task_context + +task_logs_dict = defaultdict(list) # 收集任务日志，任务id -> 日志列表 + + +class TaskLogHandler(logging.Handler): + def emit(self, record): + context: dict = getattr(record, 'jm_task_context', None) or {} # 通过 jm_task_context 字段取出任务上下文 + task_id = context.get('task_id') # 任务上下文里的task_id，就是你上面自定义的 task-1 + if task_id is not None: + task_logs_dict[task_id].append(self.format(record)) # 收集日志 + + +handler = TaskLogHandler() +handler.setFormatter(jm_logger.handlers[0].formatter) # 复用jmcomic默认 Handler 的日志格式，你也可以自定义日志格式 +jm_logger.addHandler(handler) + +# 自定义handler后，再正常使用下载方法 +task_id = 'task-1' +with jm_task_context(task_id=task_id): # 任务id + download_album(123) +``` + +任务上下文里，常用字段如下： + +| 字段 | 类型 | 谁来设置 | 含义 | 示例 | +| --- |--------| --- |-----------------------------------------------------------------------| --- | +| `task_id` | 你传入的类型；本例需可哈希，建议使用 `str` | 由你通过 `jm_task_context` 设置 | 你的一次下载任务标识 | `task-1` | +| `download_type` | `str` | jmcomic 自动设置 | 你使用的download入口函数类型，例如 download_album -> album | `album` / `photo` | +| `jm_id` | `str` | jmcomic 自动设置 | 你使用的download入口函数的入参，例如 download_album(123) -> 123。传入多个 ID 时，每个 ID 都在各自隔离的上下文中记录对应的 `jm_id`，并继承同一个 `task_id`。 | `123` | + +> 默认终端日志仅在 `task_id` 有值时显示任务上下文，并一同显示 `download_type` 和 `jm_id`。未设置 `task_id` 时，上下文仍会正常传递，但不会显示在默认日志中。 + +你也可以放入其他对象到任务上下文里，比如放入一个局部queue用来收集日志。在 jm_task_context 方法里传入即可 `jm_task_context(**fields)` + +## 5. 深度自定义：两类不同的拦截手段 根据你的需求复杂度，你可以选择以下方式： @@ -64,4 +155,4 @@ plugins: 适用于：需要完全重塑日志的分发逻辑，或者将日志直接桥接到不符合标准 logging 协议的第三方系统。 -详细参考文档：[模块自定义](./4_module_custom.md#自定义log) +代码示例：[模块自定义-自定义log](./4_module_custom.md#自定义log) diff --git a/assets/docs/sources/tutorial/13_export_and_feature.md b/assets/docs/sources/tutorial/13_export_and_feature.md index 99a63855..3829c5da 100644 --- a/assets/docs/sources/tutorial/13_export_and_feature.md +++ b/assets/docs/sources/tutorial/13_export_and_feature.md @@ -155,12 +155,12 @@ Feature (基类) └── 你的自定义 Feature ← 继承 Feature，实现任意逻辑 ``` -- **Feature 基类**：通用的附加行为抽象，不绑定任何具体实现。默认在所有生命周期钩子中执行。 +- **Feature 基类**：通用的附加行为抽象，不绑定任何具体实现。默认在所有生命周期事件中执行。 - **PluginFeature**：Feature 的子类，专门封装 jmcomic 插件。除了调用插件之外，还会根据调用来源动态适配 `filename_rule` 参数；ZIP 的打包粒度则由插件在运行时根据上下文自动推导。 ### 执行流程 -Feature **自然嵌入到 downloader 的生命周期钩子**中自动触发： +Feature **自然嵌入到 downloader 的生命周期事件**中自动触发： ```text api.download_album(extra=Feature.export_pdf) @@ -187,7 +187,7 @@ api.download_album(extra=Feature.export_pdf) > 💡 **关键点**： > -> - **执行时机**：`PluginFeature` 根据注册来源自动推导（`download_album` → `after_album`，`download_photo` → `after_photo`）。自定义 Feature 默认在所有钩子都会执行，你可以覆写 `should_invoke` 来控制。 +> - **执行时机**：`PluginFeature` 根据注册来源自动推导（`download_album` → `after_album`，`download_photo` → `after_photo`）。自定义 Feature 默认在所有事件都会执行，你可以覆写 `should_invoke` 来控制。 > - **参数自适应**：`PluginFeature` 的 `filename_rule` 前缀（A/P）会根据来源动态适配。ZIP 的打包粒度由插件根据上下文自动推导。用户显式传入的参数不会被覆盖。 ### 自定义 Feature @@ -207,4 +207,3 @@ class NotifyFeature(Feature): # 使用 download_album('123', option, extra=NotifyFeature()) ``` - diff --git a/assets/docs/sources/tutorial/4_module_custom.md b/assets/docs/sources/tutorial/4_module_custom.md index 06274c6f..d804702c 100644 --- a/assets/docs/sources/tutorial/4_module_custom.md +++ b/assets/docs/sources/tutorial/4_module_custom.md @@ -2,19 +2,21 @@ -## 自定义下载前后的回调函数 +## 自定义下载事件的回调函数 ```python -def custom_download_callback(): +def custom_download_event_callback(): """ - 该函数演示自定义下载时的回调函数 + 该函数演示如何自定义下载事件的回调函数。 + 公开下载 API 不接收 callback 参数；需要响应下载事件时，请自定义 Downloader。 """ # jmcomic的下载功能由 JmModuleConfig.CLASS_DOWNLOADER 这个类来负责执行 - # 这个类默认是 JmDownloader，继承了DownloadCallback - # 你可以写一个自定义类，继承JmDownloader，覆盖属于DownloadCallback的方法，来实现自定义回调 + # 这个类默认是 JmDownloader，继承了 DownloadCallback + # DownloadCallback 定义了 Downloader 内部各类下载事件的回调方法 + # 你可以继承 JmDownloader 并覆盖相应方法来响应事件 class MyDownloader(JmDownloader): - # 覆盖 album 下载完成后的回调 + # 覆盖 album 下载完成事件的回调函数 def after_album(self, album: JmAlbumDetail): print(f'album下载完毕: {album}') pass @@ -192,4 +194,4 @@ def custom_exception_listener(): # 注册监听器/回调 # 这个异常类（或者这个异常的子类）的实例将要被raise前，你的listener方法会被调用 JmModuleConfig.register_exception_listener(etype, listener) -``` \ No newline at end of file +``` diff --git a/assets/readme/README-en.md b/assets/readme/README-en.md index 3c813dc0..35829981 100644 --- a/assets/readme/README-en.md +++ b/assets/readme/README-en.md @@ -53,9 +53,10 @@ In addition to downloading, other JM interfaces are also implemented on demand. - Login - Search albums (supports all search parameters) -- Image downloading and decoding +- Album comments (including replies and spoiler flags) - Categories/Rankings -- Album/Chapter details +- Get album/chapter details +- Image downloading and decoding - Personal favorites - Interface encryption and decryption (for the APP API) @@ -219,9 +220,9 @@ Please check the documentation homepage → [jmcomic.readthedocs.io (Chinese lan ## Key Features -- **Supports both Async and Sync APIs** - **Bypass Cloudflare anti-bot mechanisms** - **Implement the latest decryption logic for the JM APP API (1.6.3)** +- Supports **Async** and **Sync** APIs - Multiple usages: - GitHub Actions: Enter the album ID directly on the webpage to download ([Tutorial: Download JM Albums using GitHub Actions](../docs/sources/tutorial/1_github_actions.md)) @@ -229,6 +230,7 @@ Please check the documentation homepage → [jmcomic.readthedocs.io (Chinese lan - Python Code: The most powerful usage, requiring basic Python programming knowledge - Supports both **Web** and **Mobile** implementations, switchable via configuration (**Mobile is IP restriction-free and very compatible, Web restricts some IP regions but offers higher efficiency**) - Built-in **auto-retry and domain switching** mechanisms +- Supports structured log collection by download task - **Multi-threaded downloading** (can be fine-tuned to one-thread-per-image, greatly boosting speed) - **Highly configurable** diff --git a/assets/readme/README-jp.md b/assets/readme/README-jp.md index 015d4a4b..04a057b2 100644 --- a/assets/readme/README-jp.md +++ b/assets/readme/README-jp.md @@ -53,9 +53,10 @@ - ログイン - アルバム検索（すべての検索条件をサポート） -- 画像のダウンロードおよびデコード +- アルバムコメントの取得（返信、ネタバレ表示を含む） - カテゴリ / ランキング -- アルバム / 各チャプター（エピソード）の詳細 +- アルバム / 各チャプター（エピソード）の詳細を取得 +- 画像のダウンロードおよびデコード - 個人のお気に入り - インターフェースの暗号化・復号化（APP API） @@ -214,9 +215,9 @@ jmv 350234 -y ## プロジェクトの特色 -- **Async と Sync の両方の API に対応** - **Cloudflareのボット対策のバイパス機能** - **JM APP インターフェースの最新暗号化アルゴリズムを実装 (1.6.3)** +- **Async** と **Sync** の2種類の API に対応 - 様々な利用方法のサポート： - GitHub Actions: Web上でアルバムIDを直接入力してダウンロード（[チュートリアル：GitHub Actions経由でのダウンロード](../docs/sources/tutorial/1_github_actions.md)） @@ -224,6 +225,7 @@ jmv 350234 -y - Pythonコード: 最も本質的かつ強力な利用法（一定のPythonプログラミング知識が必要です） - **Web版**と**モバイル版**の2つのクライアントをサポートし、設定から簡単に切り替え可能（**モバイル版はIP制限がなく圧倒的な互換性を持ち、Web版はエリア制限があるものの高効率**） - **自動再試行およびドメイン切り替えメカニズム**のサポート +- ダウンロードタスク単位での構造化ログ収集に対応 - **マルチスレッドダウンロード**（各画像ごとにスレッドの細分化が可能。驚異的な効率） - **強力な設定オプション** diff --git a/assets/readme/README-kr.md b/assets/readme/README-kr.md index 7a9e7ef7..03f90f41 100644 --- a/assets/readme/README-kr.md +++ b/assets/readme/README-kr.md @@ -53,9 +53,10 @@ - 로그인 - 앨범 검색 (모든 검색 매개변수 지원) -- 이미지 다운로드 및 복호화 +- 앨범 댓글 가져오기 (답글 및 스포일러 표시 포함) - 카테고리 / 랭킹 -- 앨범 / 챕터(에피소드) 세부 정보 +- 앨범 / 챕터(에피소드) 세부 정보 가져오기 +- 이미지 다운로드 및 복호화 - 개인 즐겨찾기 - 인터페이스 암호화 및 복호화 (APP API) @@ -214,9 +215,9 @@ jmv 350234 -y ## 프로젝트의 특징 -- **Async 및 Sync 두 가지 API 지원** - **Cloudflare 크롤러 방지 수칙 우회** - **금만(1.6.3) APP 인터페이스 최근의 암호화/복호화 알고리즘 완벽 지원** +- **Async** 및 **Sync** 두 가지 API 지원 - 다양한 사용 방법: - GitHub Actions: 웹페이지 내 바로 앨범 ID를 통하여 받을 수 있습니다. ([튜토리얼：GitHub Actions를 이용해 다운로드](../docs/sources/tutorial/1_github_actions.md)) @@ -224,6 +225,7 @@ jmv 350234 -y - Python 코드 구현: 가장 직관적이고 강력하며 최소한의 파이썬 기반 지식이 필요합니다. - **웹 인터페이스** 및 **모바일 인터페이스** 지원. 간단한 구성을 통한 전환 지원. (**모바일 구동환경에서의 IP제한을 돌파하고 뛰어난 호환이 됩니다, 웹은 특정 지역에서 제약받지만 성능은 최고입니다.**) - **자동 재접속(Retry) 및 도메인 전환 구조** 기본지원 +- 다운로드 작업 단위의 구조화된 로그 수집 지원 - **멀티스레드 다운로드** (1이미지 당 1스레드를 통한 극단적인 빠른 스피드) - **강력한 구성 옵션** diff --git a/requirements-dev.txt b/requirements-dev.txt index 07ed2aa1..f079b387 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,6 @@ Pillow psutil pycryptodome requests -plugin_jm_server +jm-view-server zhconv -img2pdf \ No newline at end of file +img2pdf diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 121313b6..bd3fcdfc 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,15 +2,14 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.7.2' +__version__ = '2.7.3' +from .jm_task_context import * from .api import * from .jm_plugin import * from .jm_feature import * from .jm_async_client import AsyncJmApiClient from .jm_async_downloader import JmAsyncDownloader -from .jm_client_interface import AsyncJmcomicClient -from .api import download_album_async, download_photo_async, download_batch_async # 下面进行注册组件（客户端、插件） gb = dict(filter(lambda pair: isinstance(pair[1], type), globals().items())) diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index 722d4783..b1d8ab83 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -1,26 +1,18 @@ import asyncio -import inspect from .jm_downloader import * +from .jm_task_context import bind_jm_task_context, jm_task_context __DOWNLOAD_API_RET = DownloadResult -async def _invoke_async_callback(callback, entity, downloader): - if callback is None: - return None - - is_async_callback = ( - inspect.iscoroutinefunction(callback) - or inspect.iscoroutinefunction(getattr(callback, '__call__', None)) - ) - if is_async_callback: - return await callback(entity, downloader) - - result = await asyncio.to_thread(callback, entity, downloader) - if inspect.isawaitable(result): - return await result - return result +def _download_type(download_api) -> str: + name = getattr(download_api, '__name__', download_api.__class__.__name__) + if name.endswith('_async'): + name = name[:-6] + if name.startswith('download_'): + name = name[9:] + return name def download_batch(download_api, @@ -48,21 +40,24 @@ def download_batch(download_api, result = BatchResult() + download_type = _download_type(download_api) + def _safe_download(aid): """batch 内部的单任务包装：确保异常被收集而非静默丢失""" - try: - ret = download_api(aid, option, downloader, **kwargs) - result.add(ret) - except Exception as e: - jm_log('batch.failed', f'批量下载失败: [{aid}], 异常: [{e}]', e) - result.failed[str(aid)] = e + with jm_task_context(download_type=download_type, jm_id=str(aid)): + try: + ret = download_api(aid, option, downloader, **kwargs) + result.add(ret) + except Exception as e: + jm_log('batch.failed', f'批量下载失败: [{aid}], 异常: [{e}]', e) + result.failed[str(aid)] = e multi_thread_launcher( iter_objs=set( JmcomicText.parse_to_jm_id(jmid) for jmid in jm_id_iter ), - apply_each_obj_func=_safe_download, + apply_each_obj_func=bind_jm_task_context(_safe_download), wait_finish=True ) @@ -72,7 +67,7 @@ def _safe_download(aid): def download_album(jm_album_id, option=None, downloader=None, - callback=None, + *, check_exception=True, extra=None, ) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]: @@ -84,8 +79,9 @@ def download_album(jm_album_id, :param jm_album_id: 本子的禁漫车号 :param option: 下载选项 :param downloader: 下载器类 - :param callback: 返回值回调函数，可以拿到 album 和 downloader - :param check_exception: 是否检查异常, 如果为True，会检查downloader是否有下载异常，并上抛PartialDownloadFailedException + :param check_exception: 仅当 jm_album_id 是单个 ID 时生效。为 True 时检查 downloader 中的部分下载失败， + 并上抛 PartialDownloadFailedException。多 ID 调用会转交 download_batch，此参数不生效； + 请检查 BatchResult.failed，或自行封装 download_batch 实现所需的批量异常策略。 :param extra: 下载特性（Feature），下载时动态挂载的附加行为上下文。会自动根据上下文（如 album/photo 来源）自适应参数行为。支持单个 Feature、FeatureChain、或列表 :return: 对于的本子实体类，下载器（如果是上述的批量情况，返回值为download_batch的返回值） """ @@ -93,41 +89,42 @@ def download_album(jm_album_id, if not isinstance(jm_album_id, (str, int)): return download_batch(download_album, jm_album_id, option, downloader, extra=extra) - with new_downloader(option, downloader) as dler: - # 注册 Feature 及来源，由 downloader 在 after_album 钩子中自动执行 - dler.add_features(extra, 'download_album') - album = dler.download_album(jm_album_id) + with jm_task_context(download_type='album', jm_id=str(jm_album_id)): + with new_downloader(option, downloader) as dler: + # 注册 Feature 及来源，由 downloader 在 after_album 钩子中自动执行 + dler.add_features(extra, 'download_album') + album = dler.download_album(jm_album_id) - if callback is not None: - callback(album, dler) - if check_exception: - dler.raise_if_has_exception() - return DownloadResult(album, dler) + if check_exception: + dler.raise_if_has_exception() + return DownloadResult(album, dler) def download_photo(jm_photo_id, option=None, downloader=None, - callback=None, + *, check_exception=True, extra=None, ): """ - 下载一个章节（photo），参数同 download_album + 下载一个章节（photo），参数同 download_album。 + + check_exception 仅当 jm_photo_id 是单个 ID 时生效。多 ID 场景请检查 + BatchResult.failed，或自行封装 download_batch 处理批量异常。 """ if not isinstance(jm_photo_id, (str, int)): return download_batch(download_photo, jm_photo_id, option, downloader, extra=extra) - with new_downloader(option, downloader) as dler: - # 注册 Feature 及来源，由 downloader 在 after_photo 钩子中自动执行 - dler.add_features(extra, 'download_photo') - photo = dler.download_photo(jm_photo_id) + with jm_task_context(download_type='photo', jm_id=str(jm_photo_id)): + with new_downloader(option, downloader) as dler: + # 注册 Feature 及来源，由 downloader 在 after_photo 钩子中自动执行 + dler.add_features(extra, 'download_photo') + photo = dler.download_photo(jm_photo_id) - if callback is not None: - callback(photo, dler) - if check_exception: - dler.raise_if_has_exception() - return DownloadResult(photo, dler) + if check_exception: + dler.raise_if_has_exception() + return DownloadResult(photo, dler) def new_downloader(option=None, downloader=None) -> JmDownloader: @@ -177,7 +174,7 @@ def new_async_downloader(option=None, downloader=None): async def download_album_async(jm_album_id, option=None, downloader=None, - callback=None, + *, check_exception=True, extra=None, ): @@ -185,8 +182,9 @@ async def download_album_async(jm_album_id, 异步下载一个本子（album），包含其所有的章节（photo）。 - 支持批量下载（当 jm_album_id 为可迭代对象时） - - callback 支持同步函数和异步函数 - 返回 (album, downloader) 元组，其中 downloader 的网络和线程池资源已关闭，仅用于读取下载结果 + - check_exception 仅当 jm_album_id 是单个 ID 时生效。多 ID 场景请检查 BatchResult.failed， + 或自行封装 download_batch_async 处理批量异常 """ if not isinstance(jm_album_id, (str, int)): return await download_batch_async(download_album_async, @@ -196,28 +194,29 @@ async def download_album_async(jm_album_id, extra=extra ) - async with new_async_downloader(option, downloader) as dler: - dler.add_features(extra, 'download_album') - album = await dler.download_album(jm_album_id) + with jm_task_context(download_type='album', jm_id=str(jm_album_id)): + async with new_async_downloader(option, downloader) as dler: + dler.add_features(extra, 'download_album') + album = await dler.download_album(jm_album_id) - await _invoke_async_callback(callback, album, dler) - if check_exception: - dler.raise_if_has_exception() + if check_exception: + dler.raise_if_has_exception() - return DownloadResult(album, dler) + return DownloadResult(album, dler) async def download_photo_async(jm_photo_id, option=None, downloader=None, - callback=None, + *, check_exception=True, extra=None, ): """ 异步下载一个章节（photo）。 - callback 支持同步函数和异步函数。 返回的 downloader 已关闭网络和线程池资源，仅用于读取下载结果。 + check_exception 仅当 jm_photo_id 是单个 ID 时生效。多 ID 场景请检查 + BatchResult.failed，或自行封装 download_batch_async 处理批量异常。 """ if not isinstance(jm_photo_id, (str, int)): return await download_batch_async(download_photo_async, @@ -227,15 +226,15 @@ async def download_photo_async(jm_photo_id, extra=extra ) - async with new_async_downloader(option, downloader) as dler: - dler.add_features(extra, 'download_photo') - photo = await dler.download_photo(jm_photo_id) + with jm_task_context(download_type='photo', jm_id=str(jm_photo_id)): + async with new_async_downloader(option, downloader) as dler: + dler.add_features(extra, 'download_photo') + photo = await dler.download_photo(jm_photo_id) - await _invoke_async_callback(callback, photo, dler) - if check_exception: - dler.raise_if_has_exception() + if check_exception: + dler.raise_if_has_exception() - return DownloadResult(photo, dler) + return DownloadResult(photo, dler) async def download_batch_async(download_api, @@ -253,9 +252,14 @@ async def download_batch_async(download_api, option = JmModuleConfig.option_class().default() jm_ids = list(dict.fromkeys(JmcomicText.parse_to_jm_id(jmid) for jmid in jm_id_iter)) + download_type = _download_type(download_api) + + async def _download_one(jmid): + with jm_task_context(download_type=download_type, jm_id=str(jmid)): + return await download_api(jmid, option, downloader, **kwargs) results = await asyncio.gather( - *(download_api(jmid, option, downloader, **kwargs) for jmid in jm_ids), + *(_download_one(jmid) for jmid in jm_ids), return_exceptions=True, ) @@ -263,7 +267,8 @@ async def download_batch_async(download_api, result = BatchResult() for jmid, r in zip(jm_ids, results): if isinstance(r, BaseException): - jm_log('async.batch.failed', f'批量下载失败: [{jmid}], 异常: [{r}]', r) + with jm_task_context(download_type=download_type, jm_id=str(jmid)): + jm_log('async.batch.failed', f'批量下载失败: [{jmid}], 异常: [{r}]', r) result.failed[str(jmid)] = r else: result.add(r) diff --git a/src/jmcomic/cl.py b/src/jmcomic/cl.py index 82b8868e..c5dbf311 100644 --- a/src/jmcomic/cl.py +++ b/src/jmcomic/cl.py @@ -1,9 +1,11 @@ import warnings + from .cli import * + warnings.warn( "The 'jmcomic.cl' module is deprecated and renamed to 'jmcomic.cli'. " - "Please update your imports. It will be removed in a future version.", + "Please update your imports. It will be removed in version 2.7.4.", DeprecationWarning, - stacklevel=2 + stacklevel=2, ) diff --git a/src/jmcomic/jm_async_client.py b/src/jmcomic/jm_async_client.py index 902e4762..411f4965 100644 --- a/src/jmcomic/jm_async_client.py +++ b/src/jmcomic/jm_async_client.py @@ -18,7 +18,7 @@ ) from .jm_entity import ( JmAlbumDetail, JmPhotoDetail, JmSearchPage, JmCategoryPage, - JmFavoritePage, DetailType + JmFavoritePage, DetailType, JmAlbumCommentPage ) from .jm_config import JmModuleConfig, JmMagicConstants, time_stamp, jm_log from .jm_toolkit import ( @@ -47,6 +47,7 @@ class AsyncJmApiClient(AsyncJmcomicClient): API_CHAPTER = '/chapter' API_SCRAMBLE = '/chapter_view_template' API_FAVORITE = '/favorite' + API_FORUM = '/forum' # 缓存未命中标记 _SENTINEL = object() @@ -632,6 +633,24 @@ async def favorite_folder(self, ) return JmPageTool.parse_api_to_favorite_page(resp.model_data) + async def album_pagination(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ) -> JmAlbumCommentPage: + """获取本子评论分页，返回 ``list`` 和 ``total``。""" + resp = await self.req_api( + self.API_FORUM, + params={ + 'mode': 'all', + 'page': page, + 'aid': JmcomicText.parse_to_jm_id(jm_id), + }, + ) + return JmPageTool.parse_api_to_album_comment_page(resp.model_data) + async def add_favorite_album(self, album_id, folder_id='0'): """ 将指定图集加入用户的收藏夹。 @@ -652,7 +671,7 @@ async def album_comment(self, comment_id=None, **kwargs, ) -> JmAlbumCommentResp: - """提交图集评论内容""" + """提交本子评论内容""" # 移动端 API 没有评论接口，此方法仅为接口完整性保留 raise NotImplementedError('移动端 API 不支持评论功能，请使用网页端 JmHtmlClient') diff --git a/src/jmcomic/jm_async_downloader.py b/src/jmcomic/jm_async_downloader.py index b2ff905b..ee0bef02 100644 --- a/src/jmcomic/jm_async_downloader.py +++ b/src/jmcomic/jm_async_downloader.py @@ -16,6 +16,7 @@ from .jm_entity import JmAlbumDetail, JmPhotoDetail, JmImageDetail from .jm_toolkit import JmImageTool from .jm_config import jm_log +from .jm_task_context import bind_jm_task_context from .jm_option import JmOption @@ -56,6 +57,14 @@ def __init__(self, # 核心下载流程 — 对齐 sync JmDownloader # ====================================================================== + async def _run_in_decode_pool(self, func, *args): + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + self._decode_pool, + bind_jm_task_context(func), + *args, + ) + async def download_album(self, album_id) -> JmAlbumDetail: """对齐 sync JmDownloader.download_album""" album = await self.client.get_album_detail(album_id) @@ -162,10 +171,8 @@ async def _download_single_image(self, image: JmImageDetail): img_bytes = img_resp.content # 提交到线程池解密并保存 - loop = asyncio.get_running_loop() if decode_image and image.scramble_id: - await loop.run_in_executor( - self._decode_pool, + await self._run_in_decode_pool( self._decode_and_save, img_bytes, int(image.scramble_id), @@ -183,8 +190,7 @@ async def _download_single_image(self, image: JmImageDetail): if qi != -1: img_url = img_url[:qi] need_convert = suffix_not_equal(img_url, img_save_path) - await loop.run_in_executor( - self._decode_pool, + await self._run_in_decode_pool( self._save_raw, img_bytes, img_save_path, @@ -227,28 +233,22 @@ def _save_raw(image_bytes, save_path, need_convert=False): # ====================================================================== async def before_album(self, album: JmAlbumDetail): - loop = asyncio.get_running_loop() - await loop.run_in_executor(self._decode_pool, super().before_album, album) + await self._run_in_decode_pool(super().before_album, album) async def after_album(self, album: JmAlbumDetail): - loop = asyncio.get_running_loop() - await loop.run_in_executor(self._decode_pool, super().after_album, album) + await self._run_in_decode_pool(super().after_album, album) async def before_photo(self, photo: JmPhotoDetail): - loop = asyncio.get_running_loop() - await loop.run_in_executor(self._decode_pool, super().before_photo, photo) + await self._run_in_decode_pool(super().before_photo, photo) async def after_photo(self, photo: JmPhotoDetail): - loop = asyncio.get_running_loop() - await loop.run_in_executor(self._decode_pool, super().after_photo, photo) + await self._run_in_decode_pool(super().after_photo, photo) async def before_image(self, image: JmImageDetail, img_save_path: str): - loop = asyncio.get_running_loop() - await loop.run_in_executor(self._decode_pool, super().before_image, image, img_save_path) + await self._run_in_decode_pool(super().before_image, image, img_save_path) async def after_image(self, image: JmImageDetail, img_save_path: str): - loop = asyncio.get_running_loop() - await loop.run_in_executor(self._decode_pool, super().after_image, image, img_save_path) + await self._run_in_decode_pool(super().after_image, image, img_save_path) def shutdown(self): """关闭解密线程池""" diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 8db3932d..1c7b81b4 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -1,5 +1,7 @@ from threading import Lock +from .jm_task_context import bind_jm_task_context + from .jm_client_interface import * @@ -240,6 +242,7 @@ class JmHtmlClient(AbstractJmClient): API_SEARCH = '/search/photos' API_CATEGORY = '/albums' + API_ALBUM_PAGINATION = '/ajax/album_pagination' def add_favorite_album(self, album_id, @@ -526,10 +529,37 @@ def album_comment(self, resp = self.post('/ajax/album_comment', data=data) ret = JmAlbumCommentResp(resp) + ret.require_success() jm_log('album.comment', f'{video_id}: [{comment}] ← ({ret.model().cid})') return ret + def album_pagination(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ) -> JmAlbumCommentPage: + resp = self.post( + self.API_ALBUM_PAGINATION, + data={ + 'video_id': JmcomicText.parse_to_jm_id(jm_id), + 'page': page, + 'series': series, + 'with_ad_wcm': with_ad_wcm, + } + ) + + ret = JmJsonResp(resp) + ret.require_success() + comment_page = JmPageTool.parse_html_to_album_comment_page(ret.model()) + if need_total: + album = self.get_album_detail(jm_id) + comment_page.total = album.comment_count + + return comment_page + @classmethod def require_resp_success_else_raise(cls, resp, url: str): """ @@ -608,6 +638,7 @@ class JmApiClient(AbstractJmClient): API_CHAPTER = '/chapter' API_SCRAMBLE = '/chapter_view_template' API_FAVORITE = '/favorite' + API_FORUM = '/forum' def search(self, search_query: str, @@ -854,6 +885,33 @@ def favorite_folder(self, return JmPageTool.parse_api_to_favorite_page(resp.model_data) + def album_comment(self, + video_id, + comment, + originator='', + status='true', + comment_id=None, + **kwargs, + ) -> JmAlbumCommentResp: + raise NotImplementedError('移动端 API 不支持评论功能，请使用网页端 JmHtmlClient') + + def album_pagination(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ) -> JmAlbumCommentPage: + resp = self.req_api( + self.API_FORUM, + params={ + 'mode': 'all', + 'page': page, + 'aid': JmcomicText.parse_to_jm_id(jm_id), + }, + ) + return JmPageTool.parse_api_to_album_comment_page(resp.model_data) + def add_favorite_album(self, album_id, folder_id='0', @@ -1159,7 +1217,7 @@ def get_future(self, cache_key, task): # after future done, remove it from future_dict. # cache depends on self.client instead of self.future_dict - future = self.FutureWrapper(self.executors.submit(task), + future = self.FutureWrapper(self.executors.submit(bind_jm_task_context(task)), after_done_callback=lambda: self.future_dict.pop(cache_key, None) ) diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index 387f159d..01cab739 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -144,6 +144,7 @@ def model_data(self) -> AdvancedDict: # album-comment class JmAlbumCommentResp(JmJsonResp): + @property def is_success(self) -> bool: return super().is_success and self.json()['err'] is False @@ -221,6 +222,55 @@ def album_comment(self, """ raise NotImplementedError + def album_pagination(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ) -> JmAlbumCommentPage: + """ + 获取本子评论分页。 + + 推荐直接遍历评论页，并通过 ``comment.replies`` 读取回评： + + ``` + def show_comment(comment, indent=''): + print( + indent, + comment.nickname or comment.username, + comment.content, + '剧透' if comment.is_spoiler else '非剧透', + ) + for reply in comment.replies: + show_comment(reply, indent + ' ') + + comment_page = client.album_pagination('123456', page=1) + print(comment_page.total) # 全部分页的主评论总数 + print(comment_page.comment_count) # 当前页主评论 + 所有层级回评 + for comment in comment_page: + show_comment(comment) + ``` + + 返回字段说明： + + - ``page.total``：全部分页的主评论总数。 + - ``page.comment_count``：当前页主评论加所有层级回评的数量。 + - ``len(page)``：当前页主评论数量，不包含回评。 + - ``comment.raw_data``、``page.raw_data``：统一为 ``AdvancedDict``。 + - HTML 客户端的 ``comment.likes`` 固定为 ``None``。 + - HTML 客户端的 ``comment.user_id`` 仅在头像地址包含数字 ID 时可用。 + - HTML 客户端的 ``comment.album_id`` 在评论 DOM 缺少本子链接时为 ``None``。 + + :param jm_id: album_id/photo_id + :param page: 页码，从 1 开始 + :param series: 网页端评论区的系列标记，API 端忽略 + :param with_ad_wcm: 网页端广告标记，API 端忽略 + :param need_total: 网页端是否额外请求本子详情以获取评论总数；API 端忽略 + :return: JmAlbumCommentPage + """ + raise NotImplementedError + def favorite_folder(self, page=1, order_by=JmMagicConstants.ORDER_BY_LATEST, @@ -557,6 +607,52 @@ def favorite_folder_gen(self, yield from self.do_page_iter(params, page, self.favorite_folder) + def album_pagination_gen(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ) -> Generator[JmAlbumCommentPage, None, None]: + """ + 逐页获取本子评论。 + + ``` + for comment_page in client.album_pagination_gen('123456'): + for comment in comment_page: + print(comment.content, comment.is_spoiler, len(comment.replies)) + ``` + """ + first_page = True + total = None + while True: + comment_page = self.album_pagination( + jm_id, + page=page, + series=series, + with_ad_wcm=with_ad_wcm, + need_total=need_total if first_page else False, + ) + + if first_page: + total = comment_page.total + first_page = False + else: + comment_page.total = total + + yield comment_page + + if comment_page.page_count is not None: + if page >= comment_page.page_count: + break + elif not any( + int(match.group(1)) == page + 1 + for match in JmcomicText.pattern_html_comment_next_page.finditer(comment_page.raw_html or '') + ): + break + + page += 1 + def search_gen(self, search_query: str, main_tag=0, @@ -911,6 +1007,56 @@ async def album_comment(self, ) -> JmAlbumCommentResp: raise NotImplementedError + async def album_pagination(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ) -> JmAlbumCommentPage: + """ + 获取本子评论分页。 + + 异步客户端使用移动端 API，返回值与同步 JmApiClient 一致； + ``series`` 和 ``with_ad_wcm`` 仅用于保持同步/异步签名一致。 + """ + raise NotImplementedError + + async def album_pagination_gen(self, + jm_id: str, + page=1, + series=1, + with_ad_wcm=1, + need_total=True, + ): + """异步逐页获取本子评论。""" + first_page = True + total = None + while True: + comment_page = await self.album_pagination( + jm_id, + page=page, + series=series, + with_ad_wcm=with_ad_wcm, + need_total=need_total if first_page else False, + ) + + if first_page: + total = comment_page.total + first_page = False + else: + comment_page.total = total + + yield comment_page + + if comment_page.page_count is not None: + if page >= comment_page.page_count: + break + elif not comment_page: + break + + page += 1 + # -- 域名 / 缓存管理 -- def get_domain_list(self) -> List[str]: diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index 6079d8c8..a1ad3ed5 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -1,11 +1,47 @@ from __future__ import annotations import logging + from common import time_stamp, field_cache, ProxyBuilder +from .jm_task_context import JM_TASK_CONTEXT, get_jm_task_context + jm_logger = logging.getLogger('jmcomic') +class JmLogFormatter(logging.Formatter): + """Format the stable correlation fields from a JM task context.""" + + @staticmethod + def _task_context_prefix(context) -> str: + if not isinstance(context, dict): + return '' + + fields = [] + task_id = context.get('task_id') + if task_id is None: + # 没有 task_id 时，默认不打印 task_context，保持日志简洁 + return '' + fields.append(f'task_id={task_id}') + + # 内置字段 + download_type = context.get('download_type') + jm_id = context.get('jm_id') + if download_type is not None and jm_id is not None: + fields.append(f'{download_type}={jm_id}') + elif download_type is not None: + fields.append(f'download_type={download_type}') + elif jm_id is not None: + fields.append(f'jm_id={jm_id}') + + return f'[{"; ".join(fields)}] ' if fields else '' + + def format(self, record): + # noinspection PyTypeChecker + record.jm_task_context_prefix = self._task_context_prefix(getattr(record, JM_TASK_CONTEXT.name, None)) + return super().format(record) + + def shuffled(lines): from random import shuffle from common import str_to_list @@ -19,7 +55,7 @@ def setup_default_jm_logger(): if not jm_logger.handlers: import sys handler = logging.StreamHandler(sys.stdout) - formatter = logging.Formatter('[%(asctime)s] [%(threadName)s]:【%(topic)s】%(message)s', datefmt='%Y-%m-%d %H:%M:%S') + formatter = JmLogFormatter(JmModuleConfig.VAR_LOG_FMT, datefmt='%Y-%m-%d %H:%M:%S') handler.setFormatter(formatter) jm_logger.addHandler(handler) jm_logger.setLevel(logging.INFO) @@ -30,7 +66,10 @@ def default_jm_logging(topic: str, msg, e: BaseException | None = None): if isinstance(msg, BaseException): e = msg msg = str(msg) - extra = {'topic': topic} + extra = { + 'topic': topic, + JM_TASK_CONTEXT.name: get_jm_task_context(), + } if e is not None: jm_logger.error(msg, extra=extra, exc_info=e) else: @@ -103,7 +142,7 @@ class JmMagicConstants: APP_TOKEN_SECRET_2 = '18comicAPPContent' APP_DATA_SECRET = '185Hcomic3PAPP7R' API_DOMAIN_SERVER_SECRET = 'diosfjckwpqpdfjkvnqQjsik' - APP_VERSION = '2.0.28' + APP_VERSION = '2.0.30' # 模块级别共用配置 @@ -259,6 +298,8 @@ class JmModuleConfig: # 当发生 oserror: [Errno 36] File name too long 时， # 把文件名限制在指定个字符以内 VAR_FILE_NAME_LENGTH_LIMIT = 100 + # 日志格式 + VAR_LOG_FMT = '[%(asctime)s] [%(threadName)s]:%(jm_task_context_prefix)s【%(topic)s】%(message)s' # 默认不打印 @classmethod def downloader_class(cls): @@ -565,8 +606,8 @@ def register_exception_listener(cls, etype, listener): disable_jm_log = JmModuleConfig.disable_jm_log -class PrettyFormatter(logging.Formatter): - """带 ANSI 颜色的日志格式化器，按 topic 前缀分配颜色""" +class PrettyFormatter(JmLogFormatter): + """带 ANSI 颜色的日志格式化器，按 topic 前缀分配颜色。""" TOPIC_COLORS = { 'album': '\033[1;36m', # 青色加粗 — 本子级别 @@ -581,7 +622,10 @@ class PrettyFormatter(logging.Formatter): RESET = '\033[0m' def __init__(self): - super().__init__(fmt='[%(asctime)s] %(message)s', datefmt='%H:%M:%S') + super().__init__( + fmt='[%(asctime)s] %(jm_task_context_prefix)s%(message)s', + datefmt='%H:%M:%S', + ) def format(self, record): topic = getattr(record, 'topic', '') @@ -590,7 +634,6 @@ def format(self, record): elif record.levelno >= logging.WARNING: color = self.WARN_COLOR else: - # 按 topic 前缀匹配颜色 color = next( (c for prefix, c in self.TOPIC_COLORS.items() if topic.startswith(prefix)), diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index 986196a7..f760d572 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -1,6 +1,7 @@ from typing import NamedTuple from .jm_option import * +from .jm_task_context import bind_jm_task_context def catch_exception(func): @@ -366,6 +367,8 @@ def execute_on_condition(self, if count_real == 0: return + apply = bind_jm_task_context(apply) + if count_batch >= count_real: # 一个图/章节 对应 一个线程 multi_thread_launcher( diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index bfb8d5b7..7299807b 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -651,6 +651,102 @@ def is_page(cls): return True +class JmAlbumComment(JmBaseEntity): + """ + 本子评论实体。 + + ``raw_data`` 始终为 ``AdvancedDict``。 + + 网页端字段说明： + - ``likes``：评论分页 HTML 不提供，值为 ``None``。 + - ``user_id``：仅头像地址包含数字用户 ID 时可解析，否则为 ``None``。 + - ``album_id``：评论 DOM 缺少本子链接时为 ``None``。 + """ + + def __init__(self, raw_data): + self.raw_data = AdvancedDict.wrap(raw_data or {}) + data = self.raw_data + + self.comment_id = data.get('CID') + self.album_id = data.get('AID') + self.user_id = data.get('UID') + parent_comment_id = data.get('parent_CID') + self.parent_comment_id = None if str(parent_comment_id) == '0' else parent_comment_id + self.content = data.get('content') + self.username = data.get('username') + self.nickname = data.get('nickname') + is_spoiler = data.get('is_spoiler') + if is_spoiler is None: + is_spoiler = str(data.get('spoiler')) == '2' + elif not isinstance(is_spoiler, bool): + is_spoiler = str(is_spoiler).lower() in {'1', '2', 'true'} + self.is_spoiler = is_spoiler + self.created_at = data.get('addtime') + + likes = data.get('likes') + if likes is None: + self.likes = None + else: + try: + self.likes = int(likes) + except (TypeError, ValueError): + self.likes = None + + self.replies = [ + JmAlbumComment(reply) + for reply in data.get('replys', []) or [] + ] + + +class JmAlbumCommentPage(JmBaseEntity, IndexedEntity): + """ + 本子评论分页。 + + ``total`` 是全部分页的主评论总数；``comment_count`` 是当前页主评论 + 加所有层级回评的数量；``len(page)`` 只统计当前页主评论。 + ``raw_data`` 始终为 ``AdvancedDict``。 + """ + + def __init__(self, + content: List[JmAlbumComment], + total: Optional[int] = None, + raw_html: Optional[str] = None, + raw_data=None, + ): + self.content = content + self.total = total + self.raw_html = raw_html + self.raw_data = AdvancedDict.wrap(raw_data or {}) + + @property + def page_size(self) -> int: + return 10 + + @property + def page_count(self) -> Optional[int]: + if self.total is None: + return None + + return (self.total + self.page_size - 1) // self.page_size + + @property + def comment_count(self) -> int: + def count_comment(comment): + return 1 + sum(count_comment(reply) for reply in comment.replies) + + return sum(count_comment(comment) for comment in self.content) + + def __len__(self): + return len(self.content) + + def getindex(self, index: int) -> JmAlbumComment: + return self.content[index] + + @classmethod + def is_page(cls): + return True + + class JmSearchPage(JmPageContent): @property diff --git a/src/jmcomic/jm_feature.py b/src/jmcomic/jm_feature.py index 9021d113..16d464ef 100644 --- a/src/jmcomic/jm_feature.py +++ b/src/jmcomic/jm_feature.py @@ -17,6 +17,7 @@ download_album(id, option, extra=Feature.export_pdf + Feature.export_zip) """ from .jm_plugin import * +from .jm_task_context import get_jm_task_context class Feature: @@ -33,6 +34,11 @@ class Feature: export_zip: 'PluginFeature' export_long_img: 'PluginFeature' + @property + def jm_task_context(self) -> dict: + """Return the current invocation's isolated task-context snapshot.""" + return get_jm_task_context() + def should_invoke(self, feature_from: str, when: str) -> bool: """ 判断在当前钩子(when)下，根据来源(feature_from)，是否应该执行。 diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 0373e762..93721e8c 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -3,6 +3,7 @@ """ from .jm_option import * +from .jm_task_context import bind_jm_task_context, get_jm_task_context class PluginValidationException(Exception): @@ -27,6 +28,11 @@ def invoke(self, **kwargs) -> None: """ raise NotImplementedError + @property + def jm_task_context(self) -> dict: + """Return the current invocation's isolated task-context snapshot.""" + return get_jm_task_context() + @classmethod def build(cls, option: JmOption) -> 'JmOptionPlugin': """ @@ -672,7 +678,7 @@ def main(self): # 一个收藏夹一个线程，导出收藏夹数据到文件 multi_thread_launcher( iter_objs=folders.items(), - apply_each_obj_func=self.handle_folder, + apply_each_obj_func=bind_jm_task_context(self.handle_folder), ) if not self.zip_enable: @@ -992,21 +998,34 @@ def invoke(self, if self.running: return - # 服务器的代码位于一个独立库：plugin_jm_server，需要独立安装 - # 源代码仓库：https://github.com/hect0x7/plugin-jm-server + # 服务器的代码位于独立库 jm-view-server，需要独立安装 + # 源代码仓库：https://github.com/hect0x7/jm-view-server try: # noinspection PyUnresolvedReferences - import plugin_jm_server - self.log(f'当前使用plugin_jm_server版本: {plugin_jm_server.__version__}') - except ImportError: - self.warning_lib_not_install('plugin_jm_server') - return + import jm_view_server as jm_server_lib + except ModuleNotFoundError as e: + if e.name != 'jm_view_server': + raise + + try: + # 兼容尚未迁移的 plugin_jm_server <= 0.2.3 + # noinspection PyUnresolvedReferences + import plugin_jm_server as jm_server_lib + except ModuleNotFoundError as legacy_error: + if legacy_error.name != 'plugin_jm_server': + raise + self.warning_lib_not_install('jm-view-server') + return + + self.log('检测到旧包 plugin_jm_server，建议升级到 jm-view-server', 'warning') + + self.log(f'当前使用 jm-view-server 版本: {jm_server_lib.__version__}') # 核心函数，启动服务器，会阻塞当前线程 def blocking_run_server(): self.server_thread = current_thread() self.enter_wait_list() - server = plugin_jm_server.JmServer(base_dir, password, **kwargs) + server = jm_server_lib.JmServer(base_dir, password, **kwargs) # run方法会阻塞当前线程直到flask退出 server.run(**run) diff --git a/src/jmcomic/jm_task_context.py b/src/jmcomic/jm_task_context.py new file mode 100644 index 00000000..57a5e9e1 --- /dev/null +++ b/src/jmcomic/jm_task_context.py @@ -0,0 +1,59 @@ +import inspect +from contextlib import contextmanager +from contextvars import ContextVar +from functools import wraps +from types import MappingProxyType +from typing import Callable, Mapping, Optional + + +__all__ = ( + 'JM_TASK_CONTEXT', + 'jm_task_context', + 'get_jm_task_context', + 'bind_jm_task_context', +) + + +_EMPTY_TASK_CONTEXT = MappingProxyType({}) +JM_TASK_CONTEXT: ContextVar[Mapping] = ContextVar( + 'jm_task_context', + default=_EMPTY_TASK_CONTEXT, +) + + +def get_jm_task_context() -> dict: + """Return a mutable snapshot of the current JM task context.""" + return dict(JM_TASK_CONTEXT.get()) + + +@contextmanager +def jm_task_context(**fields): + """Temporarily add fields to the current JM task context.""" + context = get_jm_task_context() + context.update(fields) + token = JM_TASK_CONTEXT.set(MappingProxyType(context)) + try: + yield + finally: + JM_TASK_CONTEXT.reset(token) + + +def bind_jm_task_context(func: Callable, context: Optional[Mapping] = None) -> Callable: + """Bind a synchronous callable to a snapshot of the current task context.""" + if (inspect.iscoroutinefunction(func) + or inspect.iscoroutinefunction(getattr(func, '__call__', None))): + raise TypeError('bind_jm_task_context only supports synchronous callables') + + snapshot = MappingProxyType(dict( + get_jm_task_context() if context is None else context + )) + + @wraps(func) + def wrapped(*args, **kwargs): + token = JM_TASK_CONTEXT.set(snapshot) + try: + return func(*args, **kwargs) + finally: + JM_TASK_CONTEXT.reset(token) + + return wrapped diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index 70dcaa1d..6b2b425b 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -1,3 +1,6 @@ +from html.parser import HTMLParser +from urllib.parse import unquote, urlparse + from PIL import Image from .jm_exception import * @@ -64,6 +67,8 @@ class JmcomicText: # 提取api接口返回值里的json，防止返回值里有无关日志导致json解析报错 pattern_api_response_json_object = compile(r'\{[\s\S]*?}') + pattern_html_comment_next_page = compile(r'id=["\']p_album_comments_\d+_(\d+)["\']') + @classmethod def parse_to_jm_domain(cls, text: str): if text.startswith(JmModuleConfig.PROT): @@ -437,6 +442,144 @@ def compare_versions(cls, v1: str, v2: str) -> int: JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env) +class HtmlTextParser(HTMLParser): + + def __init__(self): + super().__init__(convert_charrefs=True) + self.parts = [] + + def handle_starttag(self, tag, attrs): + if tag.lower() == 'br': + self.parts.append('\n') + + def handle_data(self, data): + self.parts.append(data) + + +class CommentParser(HTMLParser): + void_elements = { + 'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', + 'link', 'meta', 'param', 'source', 'track', 'wbr', + } + + def __init__(self): + super().__init__(convert_charrefs=True) + self.stack = [] + self.comments = [] + self.comment_stack = [] + + def _inside_class(self, class_name): + for frame in reversed(self.stack): + if class_name in frame['classes']: + return True + if frame['is_comment_root']: + return False + return False + + def _start_comment(self, attrs): + parent = self.comment_stack[-1] if self.comment_stack else None + comment = { + 'CID': (attrs.get('data-cid') or '').strip('{}'), + 'AID': None, + 'UID': None, + 'parent_CID': parent['CID'] if parent is not None else None, + 'content_parts': [], + 'username': None, + 'nickname_parts': [], + 'is_spoiler': False, + 'addtime_parts': [], + 'likes': None, + 'replys': [], + } + self.comment_stack.append(comment) + + def _finish_comment(self): + comment = self.comment_stack.pop() + comment['content'] = ''.join(comment.pop('content_parts')).strip() + comment['nickname'] = ''.join(comment.pop('nickname_parts')).strip() + comment['addtime'] = ''.join(comment.pop('addtime_parts')).strip() + if self.comment_stack: + self.comment_stack[-1]['replys'].append(comment) + else: + self.comments.append(comment) + + def handle_starttag(self, tag, attrs): + tag = tag.lower() + attrs = dict(attrs) + classes = set((attrs.get('class') or '').split()) + is_comment_root = ( + 'timeline' in classes + and attrs.get('data-cid') is not None + ) + self.stack.append({ + 'tag': tag, + 'classes': classes, + 'is_comment_root': is_comment_root, + }) + + if is_comment_root: + self._start_comment(attrs) + + if not self.comment_stack: + if tag in self.void_elements: + self.stack.pop() + return + + comment = self.comment_stack[-1] + + if 'disclose' in classes: + comment['is_spoiler'] = True + + if tag == 'br' and self._inside_class('timeline-content'): + comment['content_parts'].append('\n') + + if tag == 'a': + path = urlparse(attrs.get('href') or '').path + if self._inside_class('timeline-left') and '/user/' in path: + username = path.split('/user/', 1)[1].split('/', 1)[0] + comment['username'] = unquote(username) + if self._inside_class('timeline-ft') and '/photo/' in path: + comment['AID'] = path.split('/photo/', 1)[1].split('/', 1)[0] + + if tag == 'img': + path = urlparse(attrs.get('src') or '').path + if self._inside_class('timeline-left') and '/media/users/' in path: + filename = path.split('/media/users/', 1)[1].split('/', 1)[0] + user_id = filename.rsplit('.', 1)[0] + if user_id.isdigit(): + comment['UID'] = user_id + + if tag in self.void_elements: + self.stack.pop() + + def handle_endtag(self, tag): + tag = tag.lower() + for index in range(len(self.stack) - 1, -1, -1): + if self.stack[index]['tag'] != tag: + continue + + closing_frames = self.stack[index:] + del self.stack[index:] + for frame in reversed(closing_frames): + if not frame['is_comment_root']: + continue + self._finish_comment() + return + + def handle_data(self, data): + if not self.comment_stack: + return + + comment = self.comment_stack[-1] + + if self._inside_class('timeline-username'): + comment['nickname_parts'].append(data) + if self._inside_class('timeline-date'): + comment['addtime_parts'].append(data) + if self._inside_class('timeline-content'): + comment['content_parts'].append(data) + + class PatternTool: @classmethod @@ -654,6 +797,54 @@ def parse_api_to_favorite_page(cls, data: AdvancedDict) -> JmFavoritePage: return JmFavoritePage(content, folder_list, total) + @classmethod + def parse_api_to_album_comment_page(cls, data: AdvancedDict) -> JmAlbumCommentPage: + def parse_comment(item): + item_data = getattr(item, 'src_dict', item) or {} + parser = HtmlTextParser() + parser.feed(item_data.get('content') or '') + parser.close() + comment = JmAlbumComment(item) + comment.content = ''.join(parser.parts).strip() + comment.replies = [ + parse_comment(reply.raw_data) + for reply in comment.replies + ] + return comment + + content = [ + parse_comment(item) + for item in data.get('list', []) or [] + ] + + total = data.get('total') + if total is not None: + try: + total = int(total) + except (TypeError, ValueError): + total = None + + return JmAlbumCommentPage( + content=content, + total=total, + raw_data=data, + ) + + @classmethod + def parse_html_to_album_comment_page(cls, data: AdvancedDict) -> JmAlbumCommentPage: + raw_html = data.code or '' + parser = CommentParser() + parser.feed(raw_html) + parser.close() + content = [JmAlbumComment(item) for item in parser.comments] + + return JmAlbumCommentPage( + content=content, + total=None, + raw_html=raw_html, + raw_data=data, + ) + @classmethod def adapt_content(cls, content): def adapt_item(item: AdvancedDict): diff --git a/tests/test_jmcomic/test_jm_api.py b/tests/test_jmcomic/test_jm_api.py index d9af3553..bf46f99c 100644 --- a/tests/test_jmcomic/test_jm_api.py +++ b/tests/test_jmcomic/test_jm_api.py @@ -1,8 +1,44 @@ +import asyncio +import inspect +from unittest.mock import AsyncMock, patch + from test_jmcomic import * class Test_Api(JmTestConfigurable): + def test_callback_is_not_public_download_api(self): + for download_api in ( + download_album, + download_photo, + download_album_async, + download_photo_async, + ): + parameters = inspect.signature(download_api).parameters + self.assertNotIn('callback', parameters) + self.assertEqual( + parameters['check_exception'].kind, + inspect.Parameter.KEYWORD_ONLY, + ) + + def test_multi_id_shortcuts_do_not_forward_check_exception(self): + for download_api in (download_album, download_photo): + expected = BatchResult() + with patch('jmcomic.api.download_batch', return_value=expected) as batch: + actual = download_api(['123', '456'], check_exception=False) + + self.assertIs(actual, expected) + self.assertNotIn('check_exception', batch.call_args.kwargs) + + for download_api in (download_album_async, download_photo_async): + expected = BatchResult() + batch = AsyncMock(return_value=expected) + with patch('jmcomic.api.download_batch_async', new=batch): + actual = asyncio.run(download_api(['123', '456'], check_exception=False)) + + self.assertIs(actual, expected) + self.assertNotIn('check_exception', batch.call_args.kwargs) + def test_download_photo_by_id(self): """ 测试jmcomic模块的api的使用 diff --git a/tests/test_jmcomic/test_jm_async_api.py b/tests/test_jmcomic/test_jm_async_api.py index 47aa774a..995b3663 100644 --- a/tests/test_jmcomic/test_jm_async_api.py +++ b/tests/test_jmcomic/test_jm_async_api.py @@ -6,7 +6,6 @@ """ import asyncio -import threading from test_jmcomic import * from jmcomic import ( download_album_async, download_photo_async, download_batch_async, @@ -22,13 +21,6 @@ class Test_Async_Api(JmAsyncTestConfigurable): def test_async_download_photo_by_id(self): """测试 download_photo_async：验证返回值与同步版本保持一致""" photo_id = '438516' - callback_result = {} - caller_thread = threading.get_ident() - - def callback(photo, downloader): - callback_result['photo'] = photo - callback_result['downloader'] = downloader - callback_result['thread'] = threading.get_ident() # sync sync_photo, sync_dler = download_photo(photo_id, self.option) @@ -36,26 +28,16 @@ def callback(photo, downloader): async_photo, async_dler = asyncio.run(download_photo_async( photo_id, self.option, - callback=callback, )) self.assertIsInstance(async_dler, JmAsyncDownloader, 'downloader 必须是异步版本') self.assertIsInstance(async_photo, JmPhotoDetail, '返回值必须包含 photo') self.assert_sync_async_equal(sync_photo.photo_id, async_photo.photo_id, 'photo.photo_id') - self.assertIs(callback_result['photo'], async_photo) - self.assertIs(callback_result['downloader'], async_dler) - self.assertNotEqual(callback_result['thread'], caller_thread, '同步 callback 不应阻塞事件循环线程') self.assertIsNone(async_dler.client, '顶层异步下载结束后 downloader client 应已关闭') def test_async_download_album_by_id(self): """测试 download_album_async：验证返回值与同步版本保持一致""" album_id = '438516' - callback_result = {} - - async def callback(album, downloader): - await asyncio.sleep(0) - callback_result['album'] = album - callback_result['downloader'] = downloader # sync sync_album, sync_dler = download_album(album_id, self.option) @@ -63,14 +45,11 @@ async def callback(album, downloader): async_album, async_dler = asyncio.run(download_album_async( album_id, self.option, - callback=callback, )) self.assertIsInstance(async_dler, JmAsyncDownloader, 'downloader 必须是异步版本') self.assertIsInstance(async_album, JmAlbumDetail, '返回值必须包含 album') self.assert_album_equal(sync_album, async_album) - self.assertIs(callback_result['album'], async_album) - self.assertIs(callback_result['downloader'], async_dler) self.assertIsNone(async_dler.client, '顶层异步下载结束后 downloader client 应已关闭') def test_async_batch(self): diff --git a/tests/test_jmcomic/test_jm_async_client.py b/tests/test_jmcomic/test_jm_async_client.py index a7682ab2..5f0120b8 100644 --- a/tests/test_jmcomic/test_jm_async_client.py +++ b/tests/test_jmcomic/test_jm_async_client.py @@ -71,6 +71,60 @@ def test_async_comment_count(self): ) self.assertGreater(async_album.comment_count, 0, 'comment_count 应 > 0') + def test_async_album_pagination(self): + """测试异步评论分页与生成器。""" + album_id = '302820' + + async def run(): + comment_gen = self.async_client.album_pagination_gen(album_id) + page_1 = await comment_gen.__anext__() + page_2 = await comment_gen.__anext__() + await comment_gen.aclose() + + self.assertTrue(list(page_1)) + self.assertTrue(list(page_2)) + self.assertEqual(page_1.total, page_2.total) + self.assertIsInstance(page_1.raw_data, AdvancedDict) + self.assertGreaterEqual(page_1.comment_count, len(page_1)) + + comment = page_1[0] + self.assertIsInstance(comment, JmAlbumComment) + self.assertIsInstance(comment.raw_data, AdvancedDict) + self.assertIsInstance(comment.is_spoiler, bool) + self.assertTrue(all( + isinstance(reply, JmAlbumComment) + for reply in comment.replies + )) + + self.run_async(run()) + + def test_async_album_pagination_without_total(self): + """没有评论总数时，异步生成器继续读取到空页。""" + responses = iter([ + JmAlbumCommentPage([JmAlbumComment({'CID': '1'})]), + JmAlbumCommentPage([JmAlbumComment({'CID': '2'})]), + JmAlbumCommentPage([]), + ]) + original_album_pagination = self.async_client.album_pagination + + async def fake_album_pagination(*args, **kwargs): + return next(responses) + + self.async_client.album_pagination = fake_album_pagination + + async def run(): + pages = [ + page + async for page in self.async_client.album_pagination_gen('123456') + ] + self.assertEqual(len(pages), 3) + self.assertEqual([len(page) for page in pages], [1, 1, 0]) + + try: + self.run_async(run()) + finally: + self.async_client.album_pagination = original_album_pagination + def test_async_get_detail(self): """对标 test_get_detail：album + photo 联合 diff""" album_id = 400222 diff --git a/tests/test_jmcomic/test_jm_cli.py b/tests/test_jmcomic/test_jm_cli.py index 3db1b237..bdbd913d 100644 --- a/tests/test_jmcomic/test_jm_cli.py +++ b/tests/test_jmcomic/test_jm_cli.py @@ -2,7 +2,7 @@ from io import StringIO from unittest.mock import patch -from jmcomic.cl import JmcomicUI, JmViewUI +from jmcomic.cli import JmcomicUI, JmViewUI class Test_Cli(JmTestConfigurable): @@ -10,6 +10,12 @@ class Test_Cli(JmTestConfigurable): album_id = '350234' + def test_cl_deprecated(self): + with self.assertWarnsRegex(DeprecationWarning, r'removed in version 2\.7\.4'): + from jmcomic.cl import JmcomicUI as DeprecatedJmcomicUI + + self.assertIs(DeprecatedJmcomicUI, JmcomicUI) + # ========== jmcomic 命令测试 ========== def test_jmcomic_parse_album_id(self): diff --git a/tests/test_jmcomic/test_jm_client.py b/tests/test_jmcomic/test_jm_client.py index 7abc93be..400721c7 100644 --- a/tests/test_jmcomic/test_jm_client.py +++ b/tests/test_jmcomic/test_jm_client.py @@ -222,6 +222,147 @@ def test_comment_count(self): aid, ) + def test_album_pagination(self): + album_id = '302820' + api_client = self.option.new_jm_client(impl='api') + html_client = self.option.new_jm_client(impl='html') + + api_gen = api_client.album_pagination_gen(album_id) + api_page = next(api_gen) + api_page_2 = next(api_gen) + api_gen.close() + + html_page = None + failed_domains = [] + for domain in html_client.get_html_domain_all(): + html_client.set_domain_list([domain]) + try: + html_page = html_client.album_pagination(album_id, page=1) + break + except Exception as e: + failed_domains.append((domain, e)) + continue + + for domain, error in failed_domains: + print(f'本子评论请求失败，域名: {domain}, 异常: {error}') + self.assertIsNotNone( + html_page, + f'所有网页域名均无法获取本子评论，失败域名: {failed_domains}', + ) + api_comments = list(api_page) + html_comments = list(html_page) + + self.assertTrue(api_comments) + self.assertTrue(list(api_page_2)) + self.assertTrue(html_comments) + self.assertIsInstance(api_page.raw_data, AdvancedDict) + self.assertIsInstance(html_page.raw_data, AdvancedDict) + self.assertGreaterEqual(api_page.comment_count, len(api_page)) + self.assertGreaterEqual(html_page.comment_count, len(html_page)) + self.assertGreater(api_page.total, 0) + self.assertGreater(html_page.total, 0) + self.assertEqual(api_page.total, api_page_2.total) + + html_gen_without_total = html_client.album_pagination_gen( + album_id, + page=1, + need_total=False, + ) + html_page_without_total = next(html_gen_without_total) + html_page_2_without_total = next(html_gen_without_total) + html_gen_without_total.close() + + self.assertTrue(list(html_page_2_without_total)) + self.assertIsNone(html_page_without_total.total) + self.assertIsNone(html_page_without_total.page_count) + self.assertIsNone(html_page_2_without_total.total) + self.assertIsNone(html_page_2_without_total.page_count) + + for comment in (api_comments[0], html_comments[0]): + self.assertIsInstance(comment.raw_data, AdvancedDict) + self.assertTrue(comment.comment_id) + self.assertEqual(str(comment.album_id), album_id) + self.assertIsInstance(comment.content, str) + self.assertIsInstance(comment.is_spoiler, bool) + + api_comments_by_cid = { + str(comment.comment_id): comment + for comment in api_comments + } + html_comments_by_cid = { + str(comment.comment_id): comment + for comment in html_comments + } + common_cids = api_comments_by_cid.keys() & html_comments_by_cid.keys() + self.assertTrue(common_cids) + + checked_reply = False + api_pages = {1: api_page, 2: api_page_2} + for reply_page_number in range(1, 6): + candidate_api_page = api_pages.get(reply_page_number) + if candidate_api_page is None: + candidate_api_page = api_client.album_pagination( + album_id, + page=reply_page_number, + ) + candidate_html_page = ( + html_page_without_total + if reply_page_number == 1 + else html_client.album_pagination( + album_id, + page=reply_page_number, + need_total=False, + ) + ) + html_reply_comments = { + str(comment.comment_id): comment + for comment in candidate_html_page + } + for api_comment in candidate_api_page: + api_replies = api_comment.replies + html_comment = html_reply_comments.get(str(api_comment.comment_id)) + if not api_replies or html_comment is None: + continue + + html_replies = html_comment.replies + if not html_replies: + continue + + api_replies_by_cid = { + str(reply.comment_id): reply + for reply in api_replies + } + html_replies_by_cid = { + str(reply.comment_id): reply + for reply in html_replies + } + common_reply_cids = api_replies_by_cid.keys() & html_replies_by_cid.keys() + if not common_reply_cids: + continue + + reply_cid = next(iter(common_reply_cids)) + api_reply = api_replies_by_cid[reply_cid] + html_reply = html_replies_by_cid[reply_cid] + self.assertEqual(str(html_reply.comment_id), reply_cid) + self.assertEqual(str(api_reply.comment_id), reply_cid) + self.assertIsInstance(api_reply.raw_data, AdvancedDict) + self.assertIsInstance(html_reply.raw_data, AdvancedDict) + self.assertIsInstance(api_reply.content, str) + self.assertFalse(api_reply.content.startswith('