diff --git a/README.md b/README.md index 5fbb5711..c10ba851 100644 --- a/README.md +++ b/README.md @@ -220,7 +220,7 @@ jmv 350234 -y ## 项目特点 - **绕过Cloudflare的反爬虫** -- **实现禁漫APP接口最新的加解密算法 (1.6.3)** +- **实现禁漫APP接口最新的加解密算法** - 用法多样: - GitHub diff --git a/pyproject.toml b/pyproject.toml index 007c68ba..e7fdcd00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,10 +25,10 @@ classifiers=[ ] dependencies = [ "commonx>=0.6.38", - "curl-cffi", - "pillow", - "pycryptodome", - "pyyaml", + "curl-cffi>=0.5.10", + "pillow>=10.0.0", + "pycryptodome>=3.19.0", + "pyyaml>=6.0", ] dynamic = ["version"] @@ -36,6 +36,9 @@ dynamic = ["version"] Homepage = "https://github.com/hect0x7/JMComic-Crawler-Python" Documentation = "https://jmcomic.readthedocs.io" +[project.optional-dependencies] +extra = ["zhconv", "pyzipper", "py7zr", "img2pdf", "psutil"] + [project.scripts] jmcomic = "jmcomic.cli:main" jmv = "jmcomic.cli:view_main" diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index 722d4783..77a9c739 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -78,31 +78,22 @@ def download_album(jm_album_id, ) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]: """ 下载一个本子(album),包含其所有的章节(photo) - - 当jm_album_id不是str或int时,视为批量下载,相当于调用 download_batch(download_album, jm_album_id, option, downloader) - + 当jm_album_id不是str或int时,视为批量下载 :param jm_album_id: 本子的禁漫车号 :param option: 下载选项 :param downloader: 下载器类 :param callback: 返回值回调函数,可以拿到 album 和 downloader - :param check_exception: 是否检查异常, 如果为True,会检查downloader是否有下载异常,并上抛PartialDownloadFailedException - :param extra: 下载特性(Feature),下载时动态挂载的附加行为上下文。会自动根据上下文(如 album/photo 来源)自适应参数行为。支持单个 Feature、FeatureChain、或列表 - :return: 对于的本子实体类,下载器(如果是上述的批量情况,返回值为download_batch的返回值) + :param check_exception: 是否检查异常 + :param extra: 下载特性(Feature) + :return: DownloadResult (如果是批量情况返回 BatchResult) """ - if not isinstance(jm_album_id, (str, int)): return download_batch(download_album, jm_album_id, option, downloader, extra=extra) - with new_downloader(option, downloader) as dler: - # 注册 Feature 及来源,由 downloader 在 after_album 钩子中自动执行 - dler.add_features(extra, 'download_album') - album = dler.download_album(jm_album_id) - - if callback is not None: - callback(album, dler) - if check_exception: - dler.raise_if_has_exception() - return DownloadResult(album, dler) + return _download_and_return( + jm_album_id, option, downloader, callback, check_exception, extra, + 'download_album', lambda d: d.download_album(jm_album_id), + ) def download_photo(jm_photo_id, @@ -113,21 +104,35 @@ def download_photo(jm_photo_id, extra=None, ): """ - 下载一个章节(photo),参数同 download_album + 下载一个章节(photo) + 当jm_photo_id不是str或int时,视为批量下载 + :param jm_photo_id: 章节的禁漫车号 + :param option: 下载选项 + :param downloader: 下载器类 + :param callback: 返回值回调函数 + :param check_exception: 是否检查异常 + :param extra: 下载特性(Feature) """ if not isinstance(jm_photo_id, (str, int)): return download_batch(download_photo, jm_photo_id, option, downloader, extra=extra) + return _download_and_return( + jm_photo_id, option, downloader, callback, check_exception, extra, + 'download_photo', lambda d: d.download_photo(jm_photo_id), + ) + + +def _download_and_return(jm_id, option, downloader, callback, check_exception, extra, + feature_source, download_fn): with new_downloader(option, downloader) as dler: - # 注册 Feature 及来源,由 downloader 在 after_photo 钩子中自动执行 - dler.add_features(extra, 'download_photo') - photo = dler.download_photo(jm_photo_id) + dler.add_features(extra, feature_source) + entity = download_fn(dler) if callback is not None: - callback(photo, dler) + callback(entity, dler) if check_exception: dler.raise_if_has_exception() - return DownloadResult(photo, dler) + return DownloadResult(entity, dler) def new_downloader(option=None, downloader=None) -> JmDownloader: @@ -182,11 +187,9 @@ async def download_album_async(jm_album_id, extra=None, ): """ - 异步下载一个本子(album),包含其所有的章节(photo)。 - - - 支持批量下载(当 jm_album_id 为可迭代对象时) - - callback 支持同步函数和异步函数 - - 返回 (album, downloader) 元组,其中 downloader 的网络和线程池资源已关闭,仅用于读取下载结果 + 异步下载一个本子(album),包含其所有的章节(photo) + callback 支持同步函数和异步函数 + 返回的 downloader 已关闭网络和线程池资源,仅用于读取下载结果 """ if not isinstance(jm_album_id, (str, int)): return await download_batch_async(download_album_async, @@ -196,15 +199,10 @@ async def download_album_async(jm_album_id, extra=extra ) - async with new_async_downloader(option, downloader) as dler: - dler.add_features(extra, 'download_album') - album = await dler.download_album(jm_album_id) - - await _invoke_async_callback(callback, album, dler) - if check_exception: - dler.raise_if_has_exception() - - return DownloadResult(album, dler) + return await _download_async_and_return( + jm_album_id, option, downloader, callback, check_exception, extra, + 'download_album', lambda d: d.download_album(jm_album_id), + ) async def download_photo_async(jm_photo_id, @@ -215,9 +213,9 @@ async def download_photo_async(jm_photo_id, extra=None, ): """ - 异步下载一个章节(photo)。 - callback 支持同步函数和异步函数。 - 返回的 downloader 已关闭网络和线程池资源,仅用于读取下载结果。 + 异步下载一个章节(photo) + callback 支持同步函数和异步函数 + 返回的 downloader 已关闭网络和线程池资源,仅用于读取下载结果 """ if not isinstance(jm_photo_id, (str, int)): return await download_batch_async(download_photo_async, @@ -227,15 +225,23 @@ async def download_photo_async(jm_photo_id, extra=extra ) + return await _download_async_and_return( + jm_photo_id, option, downloader, callback, check_exception, extra, + 'download_photo', lambda d: d.download_photo(jm_photo_id), + ) + + +async def _download_async_and_return(jm_id, option, downloader, callback, check_exception, extra, + feature_source, download_fn): async with new_async_downloader(option, downloader) as dler: - dler.add_features(extra, 'download_photo') - photo = await dler.download_photo(jm_photo_id) + dler.add_features(extra, feature_source) + entity = await download_fn(dler) - await _invoke_async_callback(callback, photo, dler) + await _invoke_async_callback(callback, entity, dler) if check_exception: dler.raise_if_has_exception() - return DownloadResult(photo, dler) + return DownloadResult(entity, dler) async def download_batch_async(download_api, diff --git a/src/jmcomic/cli.py b/src/jmcomic/cli.py index 0e9aec6e..007fd8c5 100644 --- a/src/jmcomic/cli.py +++ b/src/jmcomic/cli.py @@ -102,6 +102,10 @@ def run(self, option): from .api import download_album, download_photo from common import MultiTaskLauncher + if len(self.album_id_list) == 0 and len(self.photo_id_list) == 0: + print('未指定任何 id,请提供 album 或 photo 的 id,例如: jmcomic 123') + return + if len(self.album_id_list) == 0: download_photo(self.photo_id_list, option) elif len(self.photo_id_list) == 0: diff --git a/src/jmcomic/jm_async_downloader.py b/src/jmcomic/jm_async_downloader.py index b2ff905b..36ab8890 100644 --- a/src/jmcomic/jm_async_downloader.py +++ b/src/jmcomic/jm_async_downloader.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio -import os from concurrent.futures import ThreadPoolExecutor from .jm_downloader import BaseDownloader @@ -50,6 +49,7 @@ def __init__(self, self._photo_semaphore = asyncio.Semaphore(photo_concurrency) # 解密线程池(CPU 密集操作卸载) + decode_worker = int(decode_worker if decode_worker is not None else option.download.threading.decode_worker) self._decode_pool = ThreadPoolExecutor(max_workers=decode_worker, thread_name_prefix='jm-async-decode') # ====================================================================== @@ -142,8 +142,9 @@ async def _download_single_image(self, image: JmImageDetail): 对齐 sync JmDownloader.download_by_image_detail 的逻辑。 """ img_save_path = self.option.decide_image_filepath(image) + from common import file_exists image.save_path = img_save_path - image.exists = os.path.exists(img_save_path) + image.exists = file_exists(img_save_path) image.cache = self.option.decide_download_cache(image) await self.before_image(image, img_save_path) diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 8db3932d..3fc61b43 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -437,13 +437,7 @@ def favorite_folder(self, return JmPageTool.parse_html_to_favorite_page(resp.text) - # noinspection PyTypeChecker - def get_username_from_cookies(self) -> str: - # cookies = self.get_meta_data('cookies', None) - # if not cookies: - # ExceptionTool.raises('未登录,无法获取到对应的用户名,请给favorite方法传入username参数') - # 解析cookies,可能需要用到 phpserialize,比较麻烦,暂不实现 - pass + def get_jm_html(self, url, require_200=True, **kwargs): """ @@ -698,12 +692,23 @@ def get_scramble_id(self, photo_id, album_id=None): if album_id is not None and album_id in cache: return cache[album_id] - scramble_id = self.fetch_scramble_id(photo_id) - cache[photo_id] = scramble_id - if album_id is not None: - cache[album_id] = scramble_id + if JmModuleConfig.SCRAMBLE_CACHE_LOCK is None: + from threading import Lock + JmModuleConfig.SCRAMBLE_CACHE_LOCK = Lock() - return scramble_id + with JmModuleConfig.SCRAMBLE_CACHE_LOCK: + # double-check after acquiring lock + if photo_id in cache: + return cache[photo_id] + if album_id is not None and album_id in cache: + return cache[album_id] + + scramble_id = self.fetch_scramble_id(photo_id) + cache[photo_id] = scramble_id + if album_id is not None: + cache[album_id] = scramble_id + + return scramble_id def fetch_detail_entity(self, jmid, clazz: Type[DetailType]) -> DetailType: """ @@ -963,16 +968,16 @@ def raise_if_resp_should_retry(self, resp, is_image): msg = JmModuleConfig.JM_ERROR_STATUS_CODE.get(code, f'HTTP状态码: {code}') ExceptionTool.raises_resp(f"禁漫API异常响应, {msg}", resp) - url = resp.request.url + url = getattr(resp, 'url', '') or getattr(getattr(resp, 'request', None), 'url', '') if self.API_SCRAMBLE in url: # /chapter_view_template 这个接口不是返回json数据,不做检查 return resp text = resp.text - for char in text: + # 只检查前1024个字符,避免遍历大型HTML页面 + for char in text[:1024]: if char not in (' ', '\n', '\t'): - # 找到第一个有效字符 ExceptionTool.require_true( char == '{', f'请求不是json格式,强制重试!响应文本: [{JmcomicText.limit_text(text, 200)}]' @@ -1099,10 +1104,12 @@ def __init__(self, future, after_done_callback): def result(self): if not self.done: - result = self.future.result() - self._result = result - self.done = True - self.future = None # help gc + try: + result = self.future.result() + self._result = result + finally: + self.done = True + self.future = None # help gc self.after_done_callback() return self._result diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index 6079d8c8..a95475f4 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -135,6 +135,7 @@ class JmModuleConfig: # 图片分隔相关 SCRAMBLE_CACHE = {} + SCRAMBLE_CACHE_LOCK = None # threading.Lock 延迟初始化 # 当本子没有作者名字时,顶替作者名字 DEFAULT_AUTHOR = 'default_author' @@ -465,6 +466,7 @@ def new_postman(cls, session=False, **kwargs): 'threading': { 'image': 30, 'photo': None, + 'decode_worker': None, }, }, 'client': { @@ -524,11 +526,14 @@ def option_default_dict(cls) -> dict: # use system proxy by default meta_data['proxies'] = cls.DEFAULT_PROXIES - # threading photo + # threading photo & decode_worker dt = option_dict['download']['threading'] if dt['photo'] is None: import os dt['photo'] = os.cpu_count() + if dt['decode_worker'] is None: + import os + dt['decode_worker'] = min(4, os.cpu_count() or 1) return option_dict @@ -619,3 +624,10 @@ def enable_pretty_log(): handler.setFormatter(PrettyFormatter()) jm_logger.addHandler(handler) jm_logger.setLevel(logging.INFO) + + +def format_album_url(aid, domain='18comic.vip'): + """ + 把album_id变为可访问的URL,方便print打印后用浏览器访问 + """ + return f'{JmModuleConfig.PROT}{domain}/album/{aid}/' diff --git a/src/jmcomic/jm_exception.py b/src/jmcomic/jm_exception.py index cb15a923..e9b97965 100644 --- a/src/jmcomic/jm_exception.py +++ b/src/jmcomic/jm_exception.py @@ -4,6 +4,7 @@ from typing import NoReturn from .jm_entity import * +from .jm_config import format_album_url class JmcomicException(Exception): @@ -156,8 +157,7 @@ def raise_missing(cls, :param resp: 响应对象 :param jmid: 禁漫本子/章节id """ - from .jm_toolkit import JmcomicText - url = JmcomicText.format_album_url(jmid) + url = format_album_url(jmid) req_type = "本子" if "album" in url else "章节" cls.raises( diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 701e198a..b507568c 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -257,6 +257,14 @@ def decide_image_save_dir(self, photo: JmPhotoDetail, ensure_exists=True) -> str photo ) + # 路径遍历防护:确保解析后的路径仍在 base_dir 范围内 + resolved = os.path.realpath(save_dir) + base = os.path.realpath(self.dir_rule.base_dir) + if not resolved.startswith(base + os.sep) and resolved != base: + raise JmcomicException( + f'路径安全校验失败: {save_dir} 不在 base_dir ({base}) 范围内' + ) + if ensure_exists: save_dir = JmcomicText.try_mkdir(save_dir) @@ -306,8 +314,7 @@ def construct(cls, origdic: Dict, cover_default=True) -> 'JmOption': # version version = dic.pop('version', None) - # noinspection PyTypeChecker - if version is not None and float(version) >= float(JmModuleConfig.JM_OPTION_VER): + if version is not None and tuple(int(x) for x in version.split('.')) >= tuple(int(x) for x in JmModuleConfig.JM_OPTION_VER.split('.')): # 版本号更高,跳过兼容代码 return cls(**dic) @@ -650,11 +657,13 @@ def call_all_plugin(self, group: str, safe=None, **extra): try: self.invoke_plugin(pclass, kwargs, extra, pinfo) - except BaseException as e: + except (KeyboardInterrupt, SystemExit): + raise + except Exception as e: if safe is True or pinfo.get('safe', True): jm_log('plugin.exception', e) else: - raise e + raise def invoke_plugin(self, pclass, kwargs: Optional[Dict], extra: dict, pinfo: dict): # 检查插件的参数类型 diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 0373e762..320bb8bf 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -71,10 +71,22 @@ def execute_deletion(self, paths: List[str]): if not self.delete_original_file: return + # 计算安全基目录:只删除 base_dir 范围内的文件 + try: + base_dir = os.path.abspath(self.option.dir_rule.base_dir) + except Exception: + base_dir = None + for p in paths: if file_not_exists(p): continue + if base_dir is not None: + real = os.path.realpath(p) + if not real.startswith(base_dir + os.sep) and real != base_dir: + self.log(f'路径不在下载目录内,跳过删除: {p}', 'remove.skip') + continue + if os.path.isdir(p): if os.listdir(p): self.log(f'文件夹中存在非本次下载的文件,请手动删除文件夹内的文件: {p}', 'remove.ignore') @@ -87,16 +99,21 @@ def execute_deletion(self, paths: List[str]): # noinspection PyMethodMayBeStatic def execute_cmd(self, cmd): - """ - 执行shell命令,这里采用简单的实现 - :param cmd: shell命令 - """ - return os.system(cmd) + import subprocess + + if isinstance(cmd, (list, tuple)): + return subprocess.run(cmd, shell=False, check=False).returncode + + return subprocess.run(cmd, shell=True, check=False).returncode # noinspection PyMethodMayBeStatic - def execute_multi_line_cmd(self, cmd: str): + def execute_multi_line_cmd(self, cmd): import subprocess - subprocess.run(cmd, shell=True, check=True) + + if isinstance(cmd, (list, tuple)): + subprocess.run(cmd, shell=False, check=True) + else: + subprocess.run(cmd, shell=True, check=True) def enter_wait_list(self): self.option.need_wait_plugins.append(self) @@ -745,16 +762,20 @@ def zip_folder_without_password(self, files, zip_path): zipf.write(file, arcname=of_file_name(file)) def zip_with_password(self): - # 构造shell命令 - cmd_list = f''' - cd {self.save_dir} - 7z a "{self.zip_filepath}" "./" -p{self.zip_password} -mhe=on > "../7z_output.txt" - - ''' - self.log(f'运行命令: {cmd_list}') - - # 执行 - self.execute_multi_line_cmd(cmd_list) + import subprocess + + save_dir = self.save_dir + zip_filepath = self.zip_filepath + zip_password = self.zip_password + + cmd = ['7z', 'a', zip_filepath, './', '-mhe=on'] + if zip_password: + cmd.append(f'-p{zip_password}') + + self.log(f'运行命令: 7z a {zip_filepath} "./" -p*** -mhe=on') + + with open(os.path.join(os.path.dirname(zip_filepath), '7z_output.txt'), 'w') as f: + subprocess.run(cmd, cwd=save_dir, check=True, stdout=f, stderr=subprocess.STDOUT) class Img2pdfPlugin(JmOptionPlugin): diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index 70dcaa1d..7af5f592 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -208,10 +208,8 @@ def format_url(cls, path, domain): @classmethod def format_album_url(cls, aid, domain='18comic.vip'): - """ - 把album_id变为可访问的URL,方便print打印后用浏览器访问 - """ - return cls.format_url(f'/album/{aid}/', domain) + from .jm_config import format_album_url as _format_album_url + return _format_album_url(aid, domain) class DSLReplacer: @@ -393,7 +391,7 @@ def try_parse_json_object(cls, resp_text: str) -> dict: @classmethod def limit_text(cls, text: str, limit: int) -> str: length = len(text) - return text if length <= limit else (text[:limit] + f'...({length - limit}') + return text if length <= limit else (text[:limit] + f'...({length - limit})') @classmethod def get_album_cover_url(cls,