Moved locks cleanup into cmapi

Author: mariadb-AlexanderPresniakov

cmapi/cmapi_server/constants.py

@@ -92,6 +92,9 @@ class ProgInfo(NamedTuple):
 LIBJEMALLOC_DEFAULT_PATH = os.path.join(MCS_DATA_PATH, 'libjemalloc.so.2')
 MCS_LOG_PATH = '/var/log/mariadb/columnstore'
 
+# tools for BRM shmem lock inspection/reset
+SHMEM_LOCKS_PATH = os.path.join(MCS_INSTALL_BIN, 'mcs-shmem-locks')
+RESET_LOCKS_PATH = os.path.join(MCS_INSTALL_BIN, 'reset_locks')
 
 # client constants
 CMAPI_PORT = 8640  #TODO: use it in all places

cmapi/cmapi_server/process_dispatchers/container.py

@@ -13,6 +13,7 @@
 from cmapi_server.constants import (
     IFLAG, LIBJEMALLOC_DEFAULT_PATH, MCS_INSTALL_BIN, ALL_MCS_PROGS,
 )
+from cmapi_server.process_dispatchers import utils as dispatcher_utils
 from cmapi_server.exceptions import CMAPIBasicError
 from cmapi_server.process_dispatchers.base import BaseDispatcher
 
@@ -223,15 +224,7 @@ def stop(
             # These stale locks can occur if the controllernode couldn't stop correctly
             #  and they cause mcs-savebrm.py to hang
 
-            logger.debug('Pre-stop: inspecting and resetting shmem locks.')
-            prestop_path = os.path.join(MCS_INSTALL_BIN, 'mcs-prestop-workernode.sh')
-            prestop_logpath = cls._create_mcs_process_logfile(
-                'mcs-prestop-workernode.log'
-            )
-            with open(prestop_logpath, 'a', encoding='utf-8') as prestop_logfh:
-                _success, _ = cls.exec_command(
-                    prestop_path, stdout=prestop_logfh
-                )
+            dispatcher_utils.reset_shmem_locks(logger)
 
             # start mcs-savebrm.py before stoping workernode
             logger.debug('Waiting to save BRM.')

cmapi/cmapi_server/process_dispatchers/systemd.py

@@ -5,6 +5,7 @@
 from typing import Union, Tuple
 
 from cmapi_server.process_dispatchers.base import BaseDispatcher
+from cmapi_server.process_dispatchers import utils as dispatcher_utils
 
 
 class SystemdDispatcher(BaseDispatcher):
@@ -164,6 +165,11 @@ def stop(
         """
         service_name = service
         if service_name == 'mcs-workernode':
+            # Run pre-stop lock reset before saving BRM
+            # These stale locks can occur if the controllernode couldn't stop correctly
+            #  and they cause mcs-savebrm.py to hang
+            dispatcher_utils.reset_shmem_locks(logging.getLogger(__name__))
+
             service_name = f'{service_name}@1.service {service_name}@2.service'
             cls._workernode_enable(False, use_sudo)
 

cmapi/cmapi_server/process_dispatchers/utils.py

@@ -0,0 +1,64 @@
+import logging
+import re
+from time import sleep
+from typing import Optional
+from cmapi_server.constants import SHMEM_LOCKS_PATH, RESET_LOCKS_PATH
+from cmapi_server.process_dispatchers.base import BaseDispatcher
+
+
+def parse_locks_num(cmd_output: str) -> int:
+    """Parse output of mcs-shmem-locks command."""
+    active_total = 0
+    for line in cmd_output.splitlines():
+        m = re.search(r'^\s*(readers|writers)\s*=\s*(\d+)', line)
+        if m:
+            try:
+                active_total += int(m.group(2))
+            except ValueError:
+                pass
+    return active_total
+
+
+def get_active_shmem_locks_num(logger: logging.Logger) -> Optional[int]:
+    """Get number of active shmem locks."""
+    cmd = f'{SHMEM_LOCKS_PATH} --lock-id 0'
+    success, out = BaseDispatcher.exec_command(cmd)
+    if not success:
+        logger.error('Failed to inspect shmem locks (command failed)')
+        return None
+    if not out:
+        logger.error('Failed to inspect shmem locks (empty output)')
+        return None
+
+    logger.debug('Current lock state:\n%s', (out or '').strip())
+
+    return parse_locks_num(out)
+
+
+def reset_shmem_locks(logger: logging.Logger) -> None:
+    """Inspect and reset BRM shmem locks"""
+    logger.debug('Inspecting and resetting shmem locks.')
+
+    # Get current lock state
+    active_locks_num = get_active_shmem_locks_num(logger)
+    if active_locks_num is None:
+        return
+
+    # Reset if any read/write locks are active
+    if active_locks_num > 0:
+        logger.info('Detected active shmem locks (sum readers+writers=%d). Attempting reset.', active_locks_num)
+
+        # Reset locks
+        success, out = BaseDispatcher.exec_command(f'{RESET_LOCKS_PATH} -s')
+        if not success:
+            logger.error('Failed to reset shmem locks (command failed)')
+            return
+
+        # Check that locks were reset
+        sleep(1)
+        active_locks_num = get_active_shmem_locks_num(logger)
+        if active_locks_num is not None and active_locks_num > 0:
+            logger.error('Failed to reset shmem locks (locks are still active)')
+            return
+    else:
+        logger.info('No active shmem locks detected.')

oam/install_scripts/CMakeLists.txt

@@ -163,7 +163,6 @@ set(SHMEM_FILE_GLOB "MCS-shm-")
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-loadbrm.py" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-savebrm.py.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcssavebrm.py" @ONLY)
-configure_file("${CMAKE_CURRENT_SOURCE_DIR}/mcs-prestop-workernode.sh.in" "${CMAKE_CURRENT_SOURCE_DIR}/mcs-prestop-workernode.sh" @ONLY)
 configure_file("${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog.in" "${CMAKE_CURRENT_SOURCE_DIR}/columnstoreSyslog" @ONLY)
 
 columnstore_install_program(columnstore-post-install ${ENGINE_BINDIR})
@@ -175,7 +174,6 @@ columnstore_install_program(columnstoreSyslogSetup.sh ${ENGINE_BINDIR})
 columnstore_install_program(mcs-stop-controllernode.sh ${ENGINE_BINDIR})
 columnstore_install_program(mcs-loadbrm.py ${ENGINE_BINDIR})
 columnstore_install_program(mcs-savebrm.py ${ENGINE_BINDIR})
-columnstore_install_program(mcs-prestop-workernode.sh ${ENGINE_BINDIR})
 columnstore_install_program(mariadb-columnstore-start.sh ${ENGINE_BINDIR})
 columnstore_install_program(mariadb-columnstore-stop.sh ${ENGINE_BINDIR})
 columnstore_install_program(loop_process_starter.sh ${ENGINE_BINDIR})

oam/install_scripts/mcs-prestop-workernode.sh.in

@@ -13,7 +13,6 @@ LimitCORE=@CORE_DUMPS@
 
 Environment="@WORKERNODE_ALLOC_CONFIG@"
 ExecStart=@ENGINE_BINDIR@/workernode DBRM_Worker%i
-ExecStop=/usr/bin/env bash -c "@ENGINE_BINDIR@/mcs-prestop-workernode.sh; /bin/kill -TERM $MAINPID"
 ExecStopPost=@ENGINE_BINDIR@/mcs-savebrm.py
 ExecStopPost=/usr/bin/env bash -c "clearShm > /dev/null 2>&1"