Bug#35854919 Client sessions not killed during shutdown, server seems stuck

During shutdown, killing connections or forcefully disconnecting them
is done based on the THDs. However, running with the threadpool plugin
may leave connections in a queue not yet handled by the thread group's
conncection handling thread, when the server starts shutting down.
Thus, the killing and forceful disconnect of the connections may be
completed before the THD for the queued connection request is created,
and hence, when the THD is eventually constructed, it will neither be
killed nor forecfully disconnected, but still, the server will wait
for the global connection count to be decremented, because the count
was incremented when the request was received by the connection event
loop. However, the counter will of course never be decremented as long
as the client keeps the connection open.

The suggested fix is to let the connection thread in each thread group
just abandon unhandled connection requests when the server has aborted
its connection event loop.

Change-Id: I7c8d2ea4407a23d4486cbd46040846a576d67d81
(cherry picked from commit 43d55d944370b5da13b8b50743f6cf7e3f97b4fd)

Author: ssorumgard

include/mysql/thread_pool_priv.h

@@ -75,6 +75,8 @@ struct Connection_handler_functions {
 THD *create_thd(Channel_info *channel_info);
 /* destroy channel_info object */
 void destroy_channel_info(Channel_info *channel_info);
+/* Is the connection events loop aborted during shutdown. */
+bool connection_events_loop_aborted();
 /* Decrement connection counter */
 void dec_connection_count();
 /*

share/messages_to_error_log.txt

@@ -12289,6 +12289,12 @@ ER_GRP_RPL_MEMBER_INFO_DOES_NOT_EXIST
 ER_USAGE_DEPRECATION_COUNTER
   eng "Deprecated '%s' used %s times, last time used '%s'."
 
+ER_WAITING_FOR_NO_CONNECTIONS
+  eng "Waiting for %u connections to be closed."
+
+ER_WAITING_FOR_NO_THDS
+  eng "Waiting for THDs in partition %u to be closed, %u still left. %u THDs left in total for all partitions."
+
 # DO NOT add server-to-client messages here;
 # they go in messages_to_clients.txt
 # in the same directory as this file.

sql/conn_handler/connection_handler_manager.cc

@@ -37,6 +37,7 @@
 #include "mysql/components/services/bits/psi_bits.h"
 #include "mysql/components/services/bits/psi_cond_bits.h"
 #include "mysql/components/services/bits/psi_mutex_bits.h"
+#include "mysql/components/services/log_builtins.h"
 #include "mysql/service_thd_wait.h"
 #include "mysqld_error.h"                              // ER_*
 #include "sql/conn_handler/channel_info.h"             // Channel_info
@@ -203,6 +204,7 @@ bool Connection_handler_manager::init() {
 void Connection_handler_manager::wait_till_no_connection() {
   mysql_mutex_lock(&LOCK_connection_count);
   while (connection_count > 0) {
+    LogErr(INFORMATION_LEVEL, ER_WAITING_FOR_NO_CONNECTIONS, connection_count);
     mysql_cond_wait(&COND_connection_count, &LOCK_connection_count);
   }
   mysql_mutex_unlock(&LOCK_connection_count);

sql/mysqld_thd_manager.cc

@@ -46,6 +46,7 @@
 #include "my_psi_config.h"
 #include "my_sys.h"
 #include "mysql/components/services/bits/psi_bits.h"
+#include "mysql/components/services/log_builtins.h"
 #include "mysql/thread_pool_priv.h"  // inc_thread_created
 #include "sql/sql_class.h"           // THD
 #include "thr_mutex.h"
@@ -278,6 +279,8 @@ void Global_THD_manager::wait_till_no_thd() {
   for (int i = 0; i < NUM_PARTITIONS; i++) {
     MUTEX_LOCK(lock, &LOCK_thd_list[i]);
     while (thd_list[i].size() > 0) {
+      LogErr(INFORMATION_LEVEL, ER_WAITING_FOR_NO_THDS, i, thd_list[i].size(),
+             get_thd_count());
       mysql_cond_wait(&COND_thd_list[i], &LOCK_thd_list[i]);
       DBUG_PRINT("quit", ("One thread died (count=%u)", get_thd_count()));
     }