Merge pull request netdata#2857 from ktsaou/master

add tcp ListenOverflows and ListenDrops chart and alarm

Author: ktsaou

conf.d/Makefile.am

@@ -102,6 +102,7 @@ dist_healthconfig_DATA = \
     health.d/softnet.conf \
     health.d/squid.conf \
     health.d/swap.conf \
+    health.d/tcp_listen.conf \
     health.d/tcp_resets.conf \
     health.d/udp_errors.conf \
     health.d/varnish.conf \

conf.d/health.d/tcp_listen.conf

@@ -0,0 +1,27 @@
+# -----------------------------------------------------------------------------
+# tcp listen sockets issues
+
+   alarm: 1m_ipv4_tcp_listen_overflows
+      on: ipv4.tcplistenissues
+      os: linux
+   hosts: *
+  lookup: sum -60s unaligned absolute of ListenOverflows
+   units: overflows
+   every: 10s
+    crit: $this > 0
+   delay: up 0 down 5m multiplier 1.5 max 1h
+    info: the number of TCP listen socket overflows during the last minute
+      to: sysadmin
+
+   alarm: 1m_ipv4_tcp_listen_drops
+      on: ipv4.tcplistenissues
+      os: linux
+   hosts: *
+  lookup: sum -60s unaligned absolute of ListenDrops
+   units: drops
+   every: 10s
+    crit: $this > 0
+   delay: up 0 down 5m multiplier 1.5 max 1h
+    info: the number of TCP listen socket drops during the last minute
+      to: sysadmin
+

conf.d/health.d/tcp_resets.conf

@@ -37,7 +37,7 @@
    every: 10s
     warn: $this > ((($1m_ipv4_tcp_resets_sent < 5)?(5):($1m_ipv4_tcp_resets_sent)) * (($status >= $WARNING)  ? (1) : (20)))
    delay: up 0 down 60m multiplier 1.2 max 2h
-options: no-clear-notification
+ options: no-clear-notification
     info: average TCP RESETS this host is sending, over the last 10 seconds (this can be an indication that a port scan is made, or that a service running on this host has crashed; clear notification for this alarm will not be sent)
       to: sysadmin
 
@@ -62,6 +62,6 @@ options: no-clear-notification
    every: 10s
     warn: $this > ((($1m_ipv4_tcp_resets_received < 5)?(5):($1m_ipv4_tcp_resets_received)) * (($status >= $WARNING)  ? (1) : (10)))
    delay: up 0 down 60m multiplier 1.2 max 2h
-options: no-clear-notification
+ options: no-clear-notification
     info: average TCP RESETS this host is receiving, over the last 10 seconds (this can be an indication that a service this host needs, has crashed; clear notification for this alarm will not be sent)
       to: sysadmin

configs.signatures

@@ -1,5 +1,6 @@
 declare -A configs_signatures=(
   ['0056936ce99788ed9ae1c611c87aa6d8']='apps_groups.conf'
+  ['007fc019fb32e952b509d455c016a002']='health.d/tcp_resets.conf'
   ['0102351817595a85d01ebd54a5f2f36b']='python.d/ovpn_status_log.conf'
   ['01302e01162d465614276de43fad7546']='python.d.conf'
   ['01c54057e0ca55b5bb49df1662d6b8c3']='python.d/web_log.conf'
@@ -485,6 +486,7 @@ declare -A configs_signatures=(
   ['f4c5d88c34d3fb853498124177cc77f1']='python.d.conf'
   ['f5736e0b2945182cb659cb0713eff923']='apps_groups.conf'
   ['f66e5236ba1245bb2e5fd99191f114c6']='charts.d/hddtemp.conf'
+  ['f68ac0fca6b4ffc96097779344cabac6']='health.d/tcp_listen.conf'
   ['f6c6656f900ff52d159dca12d624016a']='python.d/postgres.conf'
   ['f7401a6e7c7d4fe2e0e2be7f7f523275']='health.d/web_log.conf'
   ['f7a99e94231beda85c6254912d8d31c1']='python.d/tomcat.conf'

src/proc_net_netstat.c

@@ -22,7 +22,9 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
     (void)dt;
 
     static int do_bandwidth = -1, do_inerrors = -1, do_mcast = -1, do_bcast = -1, do_mcast_p = -1, do_bcast_p = -1, do_ecn = -1, \
-        do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1;
+        do_tcpext_reorder = -1, do_tcpext_syscookies = -1, do_tcpext_ofo = -1, do_tcpext_connaborts = -1, do_tcpext_memory = -1,
+        do_tcpext_listen = -1;
+
     static uint32_t hash_ipext = 0, hash_tcpext = 0;
     static procfile *ff = NULL;
 
@@ -93,6 +95,10 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
     static unsigned long long tcpext_TCPAbortOnLinger = 0;  // connections aborted after user close in linger timeout
     static unsigned long long tcpext_TCPAbortFailed = 0;    // times unable to send RST due to no memory
 
+    // https://perfchron.com/2015/12/26/investigating-linux-network-issues-with-netstat-and-nstat/
+    static unsigned long long tcpext_ListenOverflows = 0;   // times the listen queue of a socket overflowed
+    static unsigned long long tcpext_ListenDrops = 0;       // SYNs to LISTEN sockets ignored
+
     // IPv4 TCP memory pressures
     static unsigned long long tcpext_TCPMemoryPressures = 0;
 
@@ -116,6 +122,7 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
         do_tcpext_ofo        = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP out-of-order queue", CONFIG_BOOLEAN_AUTO);
         do_tcpext_connaborts = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP connection aborts", CONFIG_BOOLEAN_AUTO);
         do_tcpext_memory     = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP memory pressures", CONFIG_BOOLEAN_AUTO);
+        do_tcpext_listen     = config_get_boolean_ondemand("plugin:proc:/proc/net/netstat", "TCP listen issues", CONFIG_BOOLEAN_AUTO);
 
         arl_ipext  = arl_create("netstat/ipext", NULL, 60);
         arl_tcpext = arl_create("netstat/tcpext", NULL, 60);
@@ -197,6 +204,11 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
             arl_expect(arl_tcpext, "TCPMemoryPressures", &tcpext_TCPMemoryPressures);
         }
 
+        if(do_tcpext_listen != CONFIG_BOOLEAN_NO) {
+            arl_expect(arl_tcpext, "ListenOverflows", &tcpext_ListenOverflows);
+            arl_expect(arl_tcpext, "ListenDrops",     &tcpext_ListenDrops);
+        }
+
         // shared metrics
         arl_expect(arl_tcpext, "TCPSynRetrans", &tcpext_TCPSynRetrans);
     }
@@ -682,6 +694,40 @@ int do_proc_net_netstat(int update_every, usec_t dt) {
                 rrdset_done(st_syncookies);
             }
 
+            // --------------------------------------------------------------------
+
+            if(do_tcpext_listen == CONFIG_BOOLEAN_YES || (do_tcpext_listen == CONFIG_BOOLEAN_AUTO && (tcpext_ListenOverflows || tcpext_ListenDrops))) {
+                do_tcpext_listen = CONFIG_BOOLEAN_YES;
+
+                static RRDSET *st_listen = NULL;
+                static RRDDIM *rd_overflows = NULL, *rd_drops = NULL;
+
+                if(unlikely(!st_listen)) {
+
+                    st_listen = rrdset_create_localhost(
+                            "ipv4"
+                            , "tcplistenissues"
+                            , NULL
+                            , "tcp"
+                            , NULL
+                            , "TCP Listen Socket Issues"
+                            , "packets/s"
+                            , 3015
+                            , update_every
+                            , RRDSET_TYPE_LINE
+                    );
+
+                    rd_overflows = rrddim_add(st_listen, "ListenOverflows", "overflows",  1, 1, RRD_ALGORITHM_INCREMENTAL);
+                    rd_drops     = rrddim_add(st_listen, "ListenDrops",     "drops",      1, 1, RRD_ALGORITHM_INCREMENTAL);
+                }
+                else
+                    rrdset_next(st_listen);
+
+                rrddim_set_by_pointer(st_listen, rd_overflows, tcpext_ListenOverflows);
+                rrddim_set_by_pointer(st_listen, rd_drops,     tcpext_ListenDrops);
+
+                rrdset_done(st_listen);
+            }
         }
     }