fix multiprocess config setting always false (#855)

* fix multiprocess config setting always false

* fix process event kill

* fix bug

* enable-multiprocess support set to False

* fix bug

* add event multiprocess report

* fix bug

---------

Co-authored-by: noO0ob <[email protected]>

Dockerfile

@@ -14,8 +14,8 @@ COPY --from=nodebuilder /usr/src/lyrebird/client/ /usr/src/lyrebird/client/
 RUN if [[ -n "$USE_MIRROR" ]] ; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories ; fi \
   && apk update \
   && apk add --no-cache build-base jpeg-dev zlib-dev libffi-dev openssl-dev redis \
-  && if [[ -n "$USE_MIRROR" ]] ; then pip install --upgrade pip  -i https://pypi.douban.com/simple && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis -i https://pypi.douban.com/simple ; else pip install --upgrade pip && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis ; fi \
-  && if [[ -n "$USE_MIRROR" ]] ; then pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv -i https://pypi.douban.com/simple ; else pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv ; fi \
+  && if [[ -n "$USE_MIRROR" ]] ; then pip install --upgrade pip  -i https://pypi.tuna.tsinghua.edu.cn/simple && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis -i https://pypi.tuna.tsinghua.edu.cn/simple ; else pip install --upgrade pip && pip install --no-cache-dir . facebook-wda==0.8.1 jsonschema redis ; fi \
+  && if [[ -n "$USE_MIRROR" ]] ; then pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv -i https://pypi.tuna.tsinghua.edu.cn/simple ; else pip install werkzeug==2.2.2 mitmproxy -t /usr/local/mitmenv ; fi \
   && rm -rf /usr/src \
   && apk del --purge build-base jpeg-dev zlib-dev libffi-dev openssl-dev
 
@@ -24,7 +24,7 @@ ARG USE_MIRROR
 ENV PYTHONUNBUFFERED 1
 RUN if [[ -n "$USE_MIRROR" ]] ; then sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories ; fi \
   && apk update \
-  && apk add --no-cache jpeg zlib libffi openssl curl libstdc++ tzdata \
+  && apk add --no-cache jpeg zlib libffi openssl curl libstdc++ tzdata redis\
   && echo -e "#!/bin/sh\nexport PYTHONPATH=/usr/local/mitmenv\npython -c 'from mitmproxy.tools.main import mitmdump;mitmdump()' \$@" > /usr/local/bin/mitmdump \
   && chmod a+x /usr/local/bin/mitmdump
 COPY --from=pybuilder /usr/local/lib/python3.8/site-packages /usr/local/lib/python3.8/site-packages

lyrebird/event.py

@@ -290,7 +290,9 @@ def async_start(self):
         self.process_executor.async_obj['publish_queue'] = self.publish_server.publish_msg_queue
         self.process_executor.async_obj['eventserver'] = EventServer
         self.process_executor.start()
+        application.server['event_process_executor'] = self.process_executor
         EventServer.async_starting = True
+        self.publish('system', {'system': {'action': 'event.multiprocess', 'module': 'event_server', 'status': 'READY'}})
 
     def stop(self):
         self.publish('system', {'name': 'event.stop'})

lyrebird/manager.py

@@ -79,7 +79,7 @@ def main():
     parser.add_argument('--database', dest='database', help='Set a database path. Default is "~/.lyrebird/lyrebird.db"')
     parser.add_argument('--es', dest='extra_string', action='append', nargs=2, help='Set a custom config')
     parser.add_argument('--no-mitm', dest='no_mitm', action='store_true', help='Start without mitmproxy on 4272')
-    parser.add_argument('--enable-multiprocess', dest='enable_multiprocess', action='store_true', help='change event based on multithread to multiprocess(reply on redis)')
+    parser.add_argument('--enable-multiprocess', dest='enable_multiprocess', help='change event based on multithread to multiprocess(reply on redis)')
     parser.add_argument('--redis-port', dest='redis_port', type=int, help='specifies the redis service port currently in use, defalut is 6379')
     parser.add_argument('--redis-ip', dest='redis_ip', help='specifies the redis service ip currently in use, defalut is localhost')
     parser.add_argument('--redis-db', dest='redis_db', help='specifies the redis service db currently in use, defalut is 0')
@@ -106,9 +106,12 @@ def main():
         custom_conf['redis_port'] = args.redis_port
     if args.redis_db:
         custom_conf['redis_db'] = args.redis_db
-    if args.enable_multiprocess and compat_redis_check():
+    # --enable-multiprocess has the highest priority, 
+    # When args.enable_multiprocess is None, it is controlled by config or defaults to False
+    enable_multiprocess = args.enable_multiprocess.lower() if isinstance(args.enable_multiprocess, str) else None
+    if enable_multiprocess == 'true' and compat_redis_check():
         custom_conf['enable_multiprocess'] = True
-    else:
+    elif enable_multiprocess == 'false':
         custom_conf['enable_multiprocess'] = False
 
     application._cm = ConfigManager(conf_path_list=args.config, custom_conf=custom_conf)

lyrebird/mock/extra_mock_server/server.py

@@ -130,7 +130,7 @@ def init_app(config):
 
 async def _run_app(config):
     global semaphore
-    semaphore = asyncio.Semaphore(10) 
+    semaphore = asyncio.Semaphore(20) 
     app = init_app(config)
 
     port = config.get('extra.mock.port')

lyrebird/version.py

@@ -1,3 +1,3 @@
-IVERSION = (3, 0, 0)
+IVERSION = (3, 0, 1)
 VERSION = ".".join(str(i) for i in IVERSION)
 LYREBIRD = "Lyrebird " + VERSION