fetcher重试逻辑支持config配置

Author: a76yyyy

README.md

@@ -41,11 +41,11 @@
 
 8. **Docker已预装Curl环境, 默认安装pycurl模组**
 
-```
+```bash
 # 如需使用Proxy功能请安装PyCurl
 # Windows源码运行, 请执行 pip install pycurl==7.43.0.5 
 pip install pycurl # pip3 install pycurl
-# 如因curl导致500或599错误, 请卸载PyCurl
+# 如因curl导致500或599错误, 请卸载PyCurl或修改环境变量USE_PYCURL为False
 # pip uninstall pycurl
 ```
 
@@ -118,6 +118,12 @@ COOKIE_SECRET|否|binux|cookie加密密钥, 强烈建议修改
 PROXIES|否|""|全局代理域名列表,用"|"分隔
 PROXY_DIRECT_MODE|否|""|全局代理黑名单模式,默认不开启 <br>"url"为网址匹配模式;"regexp"为正则表达式匹配模式
 PROXY_DIRECT|否|""|全局代理黑名单匹配规则
+USE_PYCURL|否|True|是否启用Pycurl模组
+ALLOW_RETRY|否|True|在Pycurl环境下部分请求可能导致Request错误时, <br>自动修改冲突设置并重发请求
+CURL_ENCODING|否|True|是否允许使用Curl进行Encoding操作
+CURL_CONTENT_LENGTH|否|True|是否允许Curl使用Headers中自定义Content-Length请求
+NOT_RETRY_CODE|否|[详见配置](config.py)...|[详见配置](config.py)...
+empty_retry|否|True|[详见配置](config.py)...
 
 > 详细信息请查阅[config.py](config.py)
 
@@ -146,7 +152,7 @@ sh /usr/src/app/update.sh # 先进入容器后台, 执行命令后重启进程
 =========
 ## 2021.09.28 更新
 1. 更新并优化fetcher脚本
-2. 更新输入提示
+2. 更新输入提示(by [cxk000](https://github.com/a76yyyy/qiandao/commits?author=ckx000))
 3. 修复{{unicode(arg)}}不能正常转换unicode的bug
 4. 修复重复添加'|urlencode'的bug
 5. 优化fetcher重试逻辑

config.py

@@ -9,15 +9,24 @@
 import hashlib
 from urllib.parse import urlparse
 
-debug = False                                               # 是否开启Debug
-gzip = True                                                 # 是否启用gzip
-bind = str(os.getenv('BIND', '0.0.0.0'))                    # 框架运行监听地址(0.0.0.0表示监听所有IP地址)、
+debug = bool(os.getenv('QIANDAO_DEBUG',False))              # 是否开启Debug
+gzip = bool(os.getenv('GZIP',True))                         # 是否启用gzip
+bind = str(os.getenv('BIND', '0.0.0.0'))                    # 框架运行监听地址(0.0.0.0表示监听所有IP地址)
 port = int(os.getenv('PORT', 8923))                         # 监听端口Port
-https = bool(os.getenv('ENABLE_HTTPS', False))              # 发送的邮件链接启用HTTPS, 非程序使用HTTPS, 需要HTTPS需要使用反向代理
+https = bool(os.getenv('ENABLE_HTTPS', False))              # 发送的邮件链接启用HTTPS, 非框架自身HTTPS开关, 需要HTTPS请使用外部反向代理
 cookie_days = 5                                             # Cookie在客户端保留时间
 mysql_url = urlparse(os.getenv('JAWSDB_MARIA_URL', ''))     # 格式: mysql://用户名:密码@hostname:port/数据库名
 redis_url = urlparse(os.getenv('REDISCLOUD_URL', ''))       # 格式: (redis/http)://rediscloud:密码@hostname:port
 
+# PyCurl 相关设置
+use_pycurl = bool(os.getenv('USE_PYCURL',True))             # 是否启用Pycurl模组, 当环境无PyCurl模组时无效
+allow_retry = bool(os.getenv('ALLOW_RETRY', True))          # 在Pycurl环境下部分请求可能导致Request错误时, 自动修改冲突设置并重发请求
+curl_encoding = bool(os.getenv('CURL_ENCODING', True))      # 是否允许使用Curl进行Encoding操作, 当PyCurl返回"Error 61 Unrecognized transfer encoding."错误且'ALLOW_RETRY=True'时, 本次请求禁用Headers中的Content-Encoding并重试
+curl_length = bool(os.getenv('CURL_CONTENT_LENGTH', True))  # 是否允许Curl使用Headers中自定义Content-Length请求, 当PyCurl返回"HTTP 400 Bad Request"错误且'ALLOW_RETRY=True'时, 本次请求禁用Headers中的Content-Length并重试
+not_retry_code = list(map(int,os.getenv('NOT_RETRY_CODE', '301|302|303|304|305|307|400|401|403|404|405|407|408|409|410|412|415|413|414|500|501|502|503|504|599').split('|')))
+                                                            # 启用后, 当满足PyCurl启用, HTTPError code不在该列表中, 任务代理为空, 且'ALLOW_RETRY=True'时, 本次请求禁用Pycurl并重试
+empty_retry = bool(os.getenv('EMPTY_RETRY', True))          # 启用后, 当满足PyCurl启用, 返回Response为空, 任务代理为空, 且'ALLOW_RETRY=True'时, 本次请求禁用Pycurl并重试
+
 class mysql(object):
     host = mysql_url.hostname or 'localhost'                # 访问MySQL的Hostname
     port = mysql_url.port or '3306'                         # MySQL的端口Port

libs/fetcher.py

@@ -18,20 +18,23 @@
 from tornado.httputil import HTTPHeaders
 from tornado.escape import native_str
 
-try:
-    import pycurl
-except ImportError as e:
-    print(e)
-    pycurl = None
 from jinja2.sandbox import SandboxedEnvironment as Environment
-from tornado import gen, httpclient
+from tornado import gen, httpclient, simple_httpclient
 
 import config
 from libs import cookie_utils, utils
 
+if config.use_pycurl:
+    try:
+        import pycurl
+    except ImportError as e:
+        print(e)
+        pycurl = None
+else:
+    pycurl = None
+NOT_RETYR_CODE = config.not_retry_code
 logger = logging.getLogger('qiandao.fetcher')
 
-
 class Fetcher(object):
     def __init__(self, download_size_limit=config.download_size_limit):
         if pycurl:
@@ -416,25 +419,29 @@ def build_request(en):
                     version = '1.2'
                     )
                 )
-    async def build_response(self, obj, proxy={}, CURL_ENCODING=True, CURL_CONTENT_LENGTH=True):
+    async def build_response(self, obj, proxy={}, CURL_ENCODING=config.curl_encoding, CURL_CONTENT_LENGTH=config.curl_length, EMPTY_RETRY = config.empty_retry):
         try:
             req, rule, env = self.build_request(obj, download_size_limit=self.download_size_limit,proxy=proxy,CURL_ENCODING=CURL_ENCODING,CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
             response =  await gen.convert_yielded(self.client.fetch(req))
         except httpclient.HTTPError as e:
             try:
-                if e.__dict__.get('errno','') == 61:
-                    req, rule, env = self.build_request(obj, download_size_limit=self.download_size_limit,proxy=proxy,CURL_ENCODING=False)
-                    e.response =  await gen.convert_yielded(self.client.fetch(req))
-                elif e.code == 400 and e.message == 'Bad Request' and not e.response:
-                    if req and req.headers.get('content-length'):
-                        req, rule, env = self.build_request(obj, download_size_limit=self.download_size_limit,proxy=proxy,CURL_CONTENT_LENGTH=False)
+                if config.allow_retry and pycurl:
+                    if e.__dict__.get('errno','') == 61:
+                        logger.warning('{} {} [Warning] {} -> Try to retry!'.format(req.method,req.url,e))
+                        req, rule, env = self.build_request(obj, download_size_limit=self.download_size_limit,proxy=proxy,CURL_ENCODING=False,CURL_CONTENT_LENGTH=CURL_CONTENT_LENGTH)
                         e.response =  await gen.convert_yielded(self.client.fetch(req))
-                    else:
-                        httpclient.AsyncHTTPClient.configure(None)
-                        req, rule, env = self.build_request(obj, download_size_limit=self.download_size_limit,proxy=proxy)
+                    elif e.code == 400 and e.message == 'Bad Request' and req and req.headers.get('content-length'):
+                        logger.warning('{} {} [Warning] {} -> Try to retry!'.format(req.method,req.url,e))
+                        req, rule, env = self.build_request(obj, download_size_limit=self.download_size_limit,proxy=proxy,CURL_ENCODING=CURL_ENCODING,CURL_CONTENT_LENGTH=False)
                         e.response =  await gen.convert_yielded(self.client.fetch(req))
-                        if pycurl:
-                            httpclient.AsyncHTTPClient.configure('tornado.curl_httpclient.CurlAsyncHTTPClient')
+                    elif e.code not in NOT_RETYR_CODE or (EMPTY_RETRY and not e.response):
+                        logger.warning('{} {} [Warning] {} -> Try to retry!'.format(req.method,req.url,e))
+                        client = simple_httpclient.SimpleAsyncHTTPClient()
+                        e.response =  await gen.convert_yielded(client.fetch(req))
+                    else:
+                        logger.warning('{} {} [Warning] {}'.format(req.method,req.url,e))
+                else:
+                    logger.warning('{} {} [Warning] {}'.format(req.method,req.url,e))
             finally:
                 if not e.response:
                     traceback.print_exc()
@@ -443,7 +450,7 @@ async def build_response(self, obj, proxy={}, CURL_ENCODING=True, CURL_CONTENT_L
                 return rule, env, e.response
         return rule, env, response
 
-    async def fetch(self, obj, proxy={}, CURL_ENCODING=True, CURL_CONTENT_LENGTH=True):
+    async def fetch(self, obj, proxy={}, CURL_ENCODING=config.curl_encoding, CURL_CONTENT_LENGTH=config.curl_length, EMPTY_RETRY = config.empty_retry):
         """
         obj = {
           request: {
@@ -468,7 +475,7 @@ async def fetch(self, obj, proxy={}, CURL_ENCODING=True, CURL_CONTENT_LENGTH=Tru
         }
         """
 
-        rule, env, response = await gen.convert_yielded(self.build_response(obj, proxy, CURL_ENCODING, CURL_CONTENT_LENGTH))
+        rule, env, response = await gen.convert_yielded(self.build_response(obj, proxy, CURL_ENCODING, CURL_CONTENT_LENGTH, EMPTY_RETRY))
 
         env['session'].extract_cookies_to_jar(response.request, response)
         success, msg = self.run_rule(response, rule, env)