diff --git a/README.md b/README.md index ea5c111f..a9353042 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,8 @@ $ jmcomic 422866 - **可扩展性强** - 支持自定义本子/章节/图片下载前后的回调函数 - - 支持自定义日志 - 支持自定义类：`Downloader（负责调度）` `Option（负责配置）` `Client（负责请求）` `实体类`等 + - 支持自定义日志、异常监听器 - **支持Plugin插件，可以方便地扩展功能，以及使用别人的插件，目前内置插件有**： - `登录插件` - `硬件占用监控插件` diff --git a/assets/docs/sources/tutorial/4_module_custom.md b/assets/docs/sources/tutorial/4_module_custom.md index 1f9fafae..e3f84f6c 100644 --- a/assets/docs/sources/tutorial/4_module_custom.md +++ b/assets/docs/sources/tutorial/4_module_custom.md @@ -163,4 +163,32 @@ def custom_jm_log(): # 2. 让my_log生效 JmModuleConfig.log_executor = my_log +``` + + + +## 自定义异常监听器/回调 + +```python +def custom_exception_listener(): + """ + 该函数演示jmcomic的异常监听器机制 + """ + + # 1. 选一个可能会发生的、你感兴趣的异常 + etype = ResponseUnexpectedException + + + def listener(e): + """ + 你的监听器方法 + 该方法无需返回值 + :param e: 异常实例 + """ + print(f'my exception listener invoke !!! exception happened: {e}') + + + # 注册监听器/回调 + # 这个异常类（或者这个异常的子类）的实例将要被raise前，你的listener方法会被调用 + JmModuleConfig.register_exception_listener(etype, listener) ``` \ No newline at end of file diff --git a/assets/docs/sources/tutorial/8_pick_domain.md b/assets/docs/sources/tutorial/8_pick_domain.md index c94a13ec..e703922c 100644 --- a/assets/docs/sources/tutorial/8_pick_domain.md +++ b/assets/docs/sources/tutorial/8_pick_domain.md @@ -45,7 +45,7 @@ print(f'获取到{len(domain_set)}个域名，开始测试') def test_domain(domain: str): - client = option.new_jm_client(domain_list=[domain], **meta_data) + client = option.new_jm_client(impl='html', domain_list=[domain], **meta_data) status = 'ok' try: diff --git a/assets/option/option_test_html.yml b/assets/option/option_test_html.yml index 780fbc44..fbb68860 100644 --- a/assets/option/option_test_html.yml +++ b/assets/option/option_test_html.yml @@ -11,6 +11,7 @@ client: timeout: 7 domain: html: + - 18comic.org - jmcomic1.me - jmcomic.me diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 350706fc..1e68aa97 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.5.2' +__version__ = '2.5.3' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 7f71f8cb..a9d1415a 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -78,8 +78,10 @@ def request_with_retry(self, :param kwargs: 请求方法的kwargs """ if domain_index >= len(self.domain_list): - self.fallback(request, url, domain_index, retry_count, **kwargs) - + return self.fallback(request, url, domain_index, retry_count, **kwargs) + + url_backup = url + if url.startswith('/'): # path → url domain = self.domain_list[domain_index] @@ -120,9 +122,9 @@ def request_with_retry(self, self.before_retry(e, kwargs, retry_count, url) if retry_count < self.retry_times: - return self.request_with_retry(request, url, domain_index, retry_count + 1, callback, **kwargs) + return self.request_with_retry(request, url_backup, domain_index, retry_count + 1, callback, **kwargs) else: - return self.request_with_retry(request, url, domain_index + 1, 0, callback, **kwargs) + return self.request_with_retry(request, url_backup, domain_index + 1, 0, callback, **kwargs) # noinspection PyMethodMayBeStatic def raise_if_resp_should_retry(self, resp): @@ -209,7 +211,7 @@ def set_domain_list(self, domain_list: List[str]): def fallback(self, request, url, domain_index, retry_count, **kwargs): msg = f"请求重试全部失败: [{url}], {self.domain_list}" jm_log('req.fallback', msg) - ExceptionTool.raises(msg) + ExceptionTool.raises(msg, {}, RequestRetryAllFailException) # noinspection PyMethodMayBeStatic def append_params_to_url(self, url, params): diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index 1963d079..768649c8 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -117,10 +117,10 @@ class JmModuleConfig: # 移动端API域名 DOMAIN_API_LIST = str_to_list(''' + www.jmapinode.biz www.jmapinode1.top www.jmapinode2.top www.jmapinode3.top - www.jmapinode.biz www.jmapinode.top ''') @@ -144,8 +144,11 @@ class JmModuleConfig: REGISTRY_CLIENT = {} # 插件注册表 REGISTRY_PLUGIN = {} - # 异常处理器 - REGISTRY_EXCEPTION_ADVICE = {} + # 异常监听器 + # key: 异常类 + # value: 函数，参数只有异常对象，无需返回值 + # 这个异常类（或者这个异常的子类）的实例将要被raise前，你的listener方法会被调用 + REGISTRY_EXCEPTION_LISTENER = {} # 执行log的函数 executor_log = default_jm_logging @@ -311,7 +314,7 @@ def new_postman(cls, session=False, **kwargs): # 而如果只想修改几个简单常用的配置，也可以下方的DEFAULT_XXX属性 JM_OPTION_VER = '2.1' DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端 - DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存，缓存级别是level_option，详见CacheRegistry + DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理 default_option_dict: dict = { @@ -404,8 +407,8 @@ def register_client(cls, client_class): cls.REGISTRY_CLIENT[client_class.client_key] = client_class @classmethod - def register_exception_advice(cls, etype, eadvice): - cls.REGISTRY_EXCEPTION_ADVICE[etype] = eadvice + def register_exception_listener(cls, etype, listener): + cls.REGISTRY_EXCEPTION_LISTENER[etype] = listener jm_log = JmModuleConfig.jm_log diff --git a/src/jmcomic/jm_exception.py b/src/jmcomic/jm_exception.py index a75ab5ea..721fa756 100644 --- a/src/jmcomic/jm_exception.py +++ b/src/jmcomic/jm_exception.py @@ -3,9 +3,7 @@ class JmcomicException(Exception): - """ - jmcomic 模块异常 - """ + description = 'jmcomic 模块异常' def __init__(self, msg: str, context: dict): self.msg = msg @@ -16,19 +14,22 @@ def from_context(self, key): class ResponseUnexpectedException(JmcomicException): - """ - 响应不符合预期异常 - """ + description = '响应不符合预期异常' @property def resp(self): return self.from_context(ExceptionTool.CONTEXT_KEY_RESP) -class RegularNotMatchException(ResponseUnexpectedException): - """ - 正则表达式不匹配异常 - """ +class RegularNotMatchException(JmcomicException): + description = '正则表达式不匹配异常' + + @property + def resp(self): + """ + 可能为None + """ + return self.context.get(ExceptionTool.CONTEXT_KEY_RESP, None) @property def error_text(self): @@ -40,19 +41,23 @@ def pattern(self): class JsonResolveFailException(ResponseUnexpectedException): + description = 'Json解析异常' pass class MissingAlbumPhotoException(ResponseUnexpectedException): - """ - 缺少本子/章节异常 - """ + description = '不存在本子或章节异常' @property def error_jmid(self) -> str: return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID) +class RequestRetryAllFailException(JmcomicException): + description = '请求重试全部失败异常' + pass + + class ExceptionTool: """ 抛异常的工具 @@ -95,10 +100,7 @@ def raises(cls, e = etype(msg, context) # 异常处理建议 - advice = JmModuleConfig.REGISTRY_EXCEPTION_ADVICE.get(etype, None) - - if advice is not None: - advice(e) + cls.notify_all_listeners(e) raise e @@ -174,3 +176,13 @@ def new(msg, context=None, _etype=None): raises(old, msg, context) cls.raises = new + + @classmethod + def notify_all_listeners(cls, e): + registry: Dict[Type, Callable[Type]] = JmModuleConfig.REGISTRY_EXCEPTION_LISTENER + if not registry: + return None + + for accept_type, listener in registry.items(): + if isinstance(e, accept_type): + listener(e) diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 4e993767..80bcf281 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -17,8 +17,11 @@ def level_client(cls, _option, client): return registry[client] @classmethod - def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient, - cache: Union[None, bool, str, Callable]): + def enable_client_cache_on_condition(cls, + option: 'JmOption', + client: JmcomicClient, + cache: Union[None, bool, str, Callable], + ): """ cache parameter @@ -539,7 +542,7 @@ def invoke_plugin(self, pclass, kwargs: Optional[Dict], extra: dict, pinfo: dict pclass: Type[JmOptionPlugin] plugin: Optional[JmOptionPlugin] = None - + try: # 构建插件对象 plugin: JmOptionPlugin = pclass.build(self) diff --git a/tests/test_jmcomic/__init__.py b/tests/test_jmcomic/__init__.py index 79a3d4b8..79d26f63 100644 --- a/tests/test_jmcomic/__init__.py +++ b/tests/test_jmcomic/__init__.py @@ -48,7 +48,8 @@ def setUpClass(cls): # 设置 JmOption，JmcomicClient option = cls.new_option() cls.option = option - cls.client = option.build_jm_client() + # 设置缓存级别为option，可以减少请求次数 + cls.client = option.build_jm_client(cache='level_option') # 跨平台设置 cls.adapt_os() diff --git a/usage/workflow_download.py b/usage/workflow_download.py index f07428b8..9b480f0d 100644 --- a/usage/workflow_download.py +++ b/usage/workflow_download.py @@ -86,20 +86,50 @@ def log_before_raise(): jm_download_dir = env('JM_DOWNLOAD_DIR', workspace()) mkdir_if_not_exists(jm_download_dir) - # 自定义异常抛出函数，在抛出前把HTML响应数据写到下载文件夹（日志留痕） - def raises(old, msg, extra: dict): - if ExceptionTool.EXTRA_KEY_RESP not in extra: - return old(msg, extra) + def decide_filepath(e): + resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None) + + if resp is None: + suffix = str(time_stamp()) + else: + suffix = resp.url + + name = '-'.join( + fix_windir_name(it) + for it in [ + e.description, + current_thread().name, + suffix + ] + ) + + path = f'{jm_download_dir}/【出错了】{name}.log' + return path + + def exception_listener(e: JmcomicException): + """ + 异常监听器，实现了在 GitHub Actions 下，把请求错误的信息下载到文件，方便调试和通知使用者 + """ + # 决定要写入的文件路径 + path = decide_filepath(e) + + # 准备内容 + content = [ + str(type(e)), + e.msg, + ] + for k, v in e.context.items(): + content.append(f'{k}: {v}') + + # resp.text + resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None) + if resp: + content.append(f'响应文本: {resp.text}') - resp = extra[ExceptionTool.EXTRA_KEY_RESP] # 写文件 - from common import write_text, fix_windir_name - write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text) + write_text(path, '\n'.join(content)) - return old(msg, extra) - - # 应用函数 - ExceptionTool.replace_old_exception_executor(raises) + JmModuleConfig.register_exception_listener(JmcomicException, exception_listener) if __name__ == '__main__':