From c839eff31ecea1b340c012378a05709e78e34d31 Mon Sep 17 00:00:00 2001 From: Connor Osborn Date: Thu, 4 Jun 2015 17:43:36 -0700 Subject: [PATCH 1/6] intial demo tested against carbon --- graphios.py | 51 +++++++++++++++++++++++++++++++++++---- graphios_backends.py | 21 +++++++++------- plugins/plugin_example.py | 20 +++++++++++++++ setup.py | 2 +- 4 files changed, 79 insertions(+), 15 deletions(-) create mode 100644 plugins/plugin_example.py diff --git a/graphios.py b/graphios.py index 1ee2600..be0500c 100755 --- a/graphios.py +++ b/graphios.py @@ -38,6 +38,7 @@ from optparse import OptionParser import copy import graphios_backends as backends +import imp import logging import logging.handlers import os @@ -126,6 +127,7 @@ def __init__(self): self.SERVICESTATE = '' # current state afa nagios is concerned self.SERVICESTATETYPE = '' # HARD|SOFT self.GRAPHITEPREFIX = '' # graphios prefix + self.GRAPHITEPLUGIN = '' # graphios plugin path self.GRAPHITEPOSTFIX = '' # graphios suffix self.VALID = False # if this metric is valid @@ -348,6 +350,20 @@ def process_log(file_name): if mobj: # break out the metric object into one object per perfdata metric # log.debug('perfdata:%s' % mobj.PERFDATA) + # SERVICEPERFDATA::time=0.001402s;;;0.000000 size=11783B;;;0 + + if mobj.GRAPHITEPLUGIN: + module = try_load(mobj.GRAPHITEPLUGIN) + carbon_metrics = module.get_metrics(mobj.PERFDATA, mobj) + + for pair in carbon_metrics: + nobj = copy.copy(mobj) + nobj.PATH = pair[0] + nobj.VALUE = pair[1] + processed_objects.append(nobj) + + continue + for metric in mobj.PERFDATA.split(): try: nobj = copy.copy(mobj) @@ -363,6 +379,28 @@ def process_log(file_name): continue return processed_objects +def try_load(path): + """ + return the module given by path, load if not loaded + """ + base, name = os.path.split(path) + + try: + return sys.modules[name] + except KeyError: + pass + + # If any of the following calls raises an exception, + # there's a problem we can't handle -- let the caller handle it. + + fp, pathname, description = imp.find_module(name, [base] + sys.path) + + try: + return imp.load_module(name, fp, pathname, description) + finally: + # Since we may exit via an exception, close fp explicitly. + if fp: + fp.close() def get_mobj(nag_array): """ @@ -378,14 +416,17 @@ def get_mobj(nag_array): log.warn("could not split value %s, dropping metric" % var) return False - value = re.sub("/", cfg["replacement_character"], value) + replaced = re.sub("/", cfg["replacement_character"], value) + if re.search("PERFDATA", var_name): - mobj.PERFDATA = value - elif re.search("^\$_", value): + mobj.PERFDATA = replaced + elif re.search("^\$_", replaced): continue + elif re.search("PLUGIN", var_name): + mobj.GRAPHITEPLUGIN = value else: - value = re.sub("\s", "", value) - setattr(mobj, var_name, value) + setattr(mobj, var_name, replaced) + mobj.validate() if mobj.VALID is True: return mobj diff --git a/graphios_backends.py b/graphios_backends.py index 81e976a..f9bdaf0 100644 --- a/graphios_backends.py +++ b/graphios_backends.py @@ -312,17 +312,19 @@ def build_path(self, m): """ Builds a carbon metric """ - if m.GRAPHITEPREFIX != "": + if m.GRAPHITEPLUGIN: + return m.PATH + + path = "" + pre = "" + post = "" + + if m.GRAPHITEPREFIX: pre = "%s." % m.GRAPHITEPREFIX - else: - pre = "" - if m.GRAPHITEPOSTFIX != "": + + if m.GRAPHITEPOSTFIX: post = ".%s" % m.GRAPHITEPOSTFIX - else: - post = "" - # if self.replace_hostname: - # hostname = m.HOSTNAME.replace('.', self.replacement_character) - # else: + hostname = m.HOSTNAME if self.use_service_desc: # we want: (prefix.)hostname.service_desc(.postfix).perfdata @@ -331,6 +333,7 @@ def build_path(self, m): m.LABEL) else: path = "%s%s%s.%s" % (pre, hostname, post, m.LABEL) + path = re.sub(r"\.$", '', path) # fix paths that end in dot path = re.sub(r"\.\.", '.', path) # fix paths with double dots path = self.fix_string(path) diff --git a/plugins/plugin_example.py b/plugins/plugin_example.py new file mode 100644 index 0000000..0467cac --- /dev/null +++ b/plugins/plugin_example.py @@ -0,0 +1,20 @@ +# This is a sample plugin used for flexible reporting of metrics +# +# Graphios makes the call `get_metrics(perfdata, nag)` where +# +# nag: contains all the fields defined in the nagios log +# perfdata: is just nag.PERFDATA +# +# It expects the output to be a list of metric paths and values and +# Graphios will use the nag.TIMET timestamp to form the correct metric +# + +def get_metrics(perfdata, nag): + """ + returns a [(, )] where each is the metric to send to carbon + """ + + path = "%s.%s.%s.%s" % (nag.GRAPHITEPREFIX, nag.HOSTNAME, nag.GRAPHITEPOSTFIX, nag.LABEL) + value = nag.VALUE + + return [ (path, value) ] diff --git a/setup.py b/setup.py index 9b2edb1..16382fe 100644 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ def _post_install(): """ tries to find the nagios.cfg and insert graphios perf commands/cfg """ - lookin = ['/etc/nagios/', '/opt/nagios/', '/usr/local/nagios', + lookin = ['/etc/nagios3/','/etc/nagios/', '/opt/nagios/', '/usr/local/nagios', '/usr/nagios'] nag_cfg = find_nagios_cfg(lookin) if nag_cfg is None: From 5c0d3c32fd3e0373428e44e2b9236a8e39872efb Mon Sep 17 00:00:00 2001 From: Connor Osborn Date: Mon, 15 Jun 2015 10:25:05 -0700 Subject: [PATCH 2/6] Allow plugin in nagios definition include py extension --- graphios.py | 1 + 1 file changed, 1 insertion(+) diff --git a/graphios.py b/graphios.py index be0500c..334813a 100755 --- a/graphios.py +++ b/graphios.py @@ -384,6 +384,7 @@ def try_load(path): return the module given by path, load if not loaded """ base, name = os.path.split(path) + name, ext = os.path.splitext(name) try: return sys.modules[name] From 37cd8c5faa783ae2b112e9e4cf9a81ef447e61b2 Mon Sep 17 00:00:00 2001 From: Connor Osborn Date: Mon, 15 Jun 2015 11:23:28 -0700 Subject: [PATCH 3/6] Updated definitions to include plugin variable --- nagios/nagios_perfdata.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nagios/nagios_perfdata.cfg b/nagios/nagios_perfdata.cfg index 35c47ae..59a4ddc 100644 --- a/nagios/nagios_perfdata.cfg +++ b/nagios/nagios_perfdata.cfg @@ -2,12 +2,12 @@ ###### Auto-generated Graphios configs ####### process_performance_data=1 service_perfdata_file=/var/spool/nagios/graphios/service-perfdata -service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$\tGRAPHITEPREFIX::$_SERVICEGRAPHITEPREFIX$\tGRAPHITEPOSTFIX::$_SERVICEGRAPHITEPOSTFIX$\tMETRICTYPE::$_SERVICEMETRICTYPE$ +service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tSERVICEOUTPUT::$SERVICEOUTPUT$\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$\tGRAPHITEPREFIX::$_SERVICEGRAPHITEPREFIX$\tGRAPHITEPLUGIN::$_SERVICEGRAPHITEPLUGIN$\tGRAPHITEPOSTFIX::$_SERVICEGRAPHITEPOSTFIX$\tMETRICTYPE::$_SERVICEMETRICTYPE$ service_perfdata_file_mode=a service_perfdata_file_processing_interval=15 service_perfdata_file_processing_command=graphios_perf_service host_perfdata_file=/var/spool/nagios/graphios/host-perfdata -host_perfdata_file_template=DATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tGRAPHITEPREFIX::$_HOSTGRAPHITEPREFIX$\tGRAPHITEPOSTFIX::$_HOSTGRAPHITEPOSTFIX$\tMETRICTYPE::$_HOSTMETRICTYPE$ +host_perfdata_file_template=GRAPHITEPLUGIN::$_HOSTGRAPHITEPLUGIN$\tDATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tGRAPHITEPREFIX::$_HOSTGRAPHITEPREFIX$\tGRAPHITEPOSTFIX::$_HOSTGRAPHITEPOSTFIX$\tMETRICTYPE::$_HOSTMETRICTYPE$ host_perfdata_file_mode=a host_perfdata_file_processing_interval=15 host_perfdata_file_processing_command=graphios_perf_host From eb9698bae0615a4e7825ff2bcc44d022b2e44643 Mon Sep 17 00:00:00 2001 From: Connor Osborn Date: Mon, 29 Jun 2015 11:26:52 -0700 Subject: [PATCH 4/6] Clean spacing/Error handling --- graphios.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/graphios.py b/graphios.py index 334813a..4314d0c 100755 --- a/graphios.py +++ b/graphios.py @@ -379,6 +379,7 @@ def process_log(file_name): continue return processed_objects + def try_load(path): """ return the module given by path, load if not loaded @@ -398,11 +399,17 @@ def try_load(path): try: return imp.load_module(name, fp, pathname, description) + except (IOError, OSError, Exception) as e: + print "Exception '%s' loading plugin: %s" % (e, path) + print "Check if dir exists, or file permissions." + print "Exiting." + exit(1) finally: # Since we may exit via an exception, close fp explicitly. if fp: fp.close() + def get_mobj(nag_array): """ takes a split array of nagios variables and returns a mobj if it's From c9499048fcc3b0470e4131a995396cc9869b19ae Mon Sep 17 00:00:00 2001 From: Connor Osborn Date: Wed, 1 Jul 2015 11:25:25 -0700 Subject: [PATCH 5/6] Replace nagios variable with config variable --- graphios.py | 42 ++++++++++++++++++++++---------------- graphios_backends.py | 9 +++++++- nagios/nagios_perfdata.cfg | 4 ++-- plugins/plugin_example.py | 21 ++++++++++++++++--- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/graphios.py b/graphios.py index 4314d0c..6d90e88 100755 --- a/graphios.py +++ b/graphios.py @@ -127,7 +127,6 @@ def __init__(self): self.SERVICESTATE = '' # current state afa nagios is concerned self.SERVICESTATETYPE = '' # HARD|SOFT self.GRAPHITEPREFIX = '' # graphios prefix - self.GRAPHITEPLUGIN = '' # graphios plugin path self.GRAPHITEPOSTFIX = '' # graphios suffix self.VALID = False # if this metric is valid @@ -218,7 +217,8 @@ def verify_config(config_dict): global spool_directory ensure_list = ['replacement_character', 'log_file', 'log_max_size', 'log_level', 'sleep_time', 'sleep_max', 'test_mode', - 'reverse_hostname', 'replace_hostname'] + 'reverse_hostname', 'replace_hostname', 'plugin_path', + 'override_with_plugin'] missing_values = [] for ensure in ensure_list: if ensure not in config_dict: @@ -230,6 +230,18 @@ def verify_config(config_dict): sys.exit(1) if "spool_directory" in config_dict: spool_directory = config_dict['spool_directory'] + if config_dict['override_with_plugin']: + plugin = try_load(config_dict['plugin_path']) + try: + plugin.get_metrics + except Exception as e: + print "Exception occured while loading plugin path %s:" \ + % (config_dict['plugin_path']) + print " %s" % e + print "Plugin must implement 'get_metrics' see \ + plugins/plugin_example.py" + print "Exiting." + exit(1) def print_debug(msg): @@ -352,8 +364,8 @@ def process_log(file_name): # log.debug('perfdata:%s' % mobj.PERFDATA) # SERVICEPERFDATA::time=0.001402s;;;0.000000 size=11783B;;;0 - if mobj.GRAPHITEPLUGIN: - module = try_load(mobj.GRAPHITEPLUGIN) + if cfg['override_with_plugin']: + module = try_load(cfg['plugin_path']) carbon_metrics = module.get_metrics(mobj.PERFDATA, mobj) for pair in carbon_metrics: @@ -387,27 +399,20 @@ def try_load(path): base, name = os.path.split(path) name, ext = os.path.splitext(name) + # Return module if already loaded try: return sys.modules[name] except KeyError: pass - # If any of the following calls raises an exception, - # there's a problem we can't handle -- let the caller handle it. - - fp, pathname, description = imp.find_module(name, [base] + sys.path) - try: - return imp.load_module(name, fp, pathname, description) + return imp.load_source(name, path) except (IOError, OSError, Exception) as e: - print "Exception '%s' loading plugin: %s" % (e, path) - print "Check if dir exists, or file permissions." + print "Exception occured while loading plugin path %s:" % path + print " %s" % e + print "Check the path, or file permissions." print "Exiting." exit(1) - finally: - # Since we may exit via an exception, close fp explicitly. - if fp: - fp.close() def get_mobj(nag_array): @@ -430,9 +435,10 @@ def get_mobj(nag_array): mobj.PERFDATA = replaced elif re.search("^\$_", replaced): continue - elif re.search("PLUGIN", var_name): - mobj.GRAPHITEPLUGIN = value + elif cfg['override_with_plugin']: + setattr(mobj, var_name, value) else: + replaced = re.sub("\s", "", replaced) setattr(mobj, var_name, replaced) mobj.validate() diff --git a/graphios_backends.py b/graphios_backends.py index f9bdaf0..83da6ea 100644 --- a/graphios_backends.py +++ b/graphios_backends.py @@ -274,6 +274,13 @@ def __init__(self, cfg): except: self.carbon_plaintext = False + try: + if cfg['override_with_plugin']: + self.plugin_path = cfg['plugin_path'] + self.override_with_plugin = cfg['override_with_plugin'] + except: + self.override_with_plugin = False + def convert_messages(self, metrics): """ Converts the metric obj list into graphite messages @@ -312,7 +319,7 @@ def build_path(self, m): """ Builds a carbon metric """ - if m.GRAPHITEPLUGIN: + if self.override_with_plugin: return m.PATH path = "" diff --git a/nagios/nagios_perfdata.cfg b/nagios/nagios_perfdata.cfg index 59a4ddc..7325242 100644 --- a/nagios/nagios_perfdata.cfg +++ b/nagios/nagios_perfdata.cfg @@ -2,12 +2,12 @@ ###### Auto-generated Graphios configs ####### process_performance_data=1 service_perfdata_file=/var/spool/nagios/graphios/service-perfdata -service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tSERVICEOUTPUT::$SERVICEOUTPUT$\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$\tGRAPHITEPREFIX::$_SERVICEGRAPHITEPREFIX$\tGRAPHITEPLUGIN::$_SERVICEGRAPHITEPLUGIN$\tGRAPHITEPOSTFIX::$_SERVICEGRAPHITEPOSTFIX$\tMETRICTYPE::$_SERVICEMETRICTYPE$ +service_perfdata_file_template=DATATYPE::SERVICEPERFDATA\tSERVICEOUTPUT::$SERVICEOUTPUT$\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tSERVICEDESC::$SERVICEDESC$\tSERVICEPERFDATA::$SERVICEPERFDATA$\tSERVICECHECKCOMMAND::$SERVICECHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tSERVICESTATE::$SERVICESTATE$\tSERVICESTATETYPE::$SERVICESTATETYPE$\tGRAPHITEPREFIX::$_SERVICEGRAPHITEPREFIX$\tGRAPHITEPOSTFIX::$_SERVICEGRAPHITEPOSTFIX$\tMETRICTYPE::$_SERVICEMETRICTYPE$ service_perfdata_file_mode=a service_perfdata_file_processing_interval=15 service_perfdata_file_processing_command=graphios_perf_service host_perfdata_file=/var/spool/nagios/graphios/host-perfdata -host_perfdata_file_template=GRAPHITEPLUGIN::$_HOSTGRAPHITEPLUGIN$\tDATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tGRAPHITEPREFIX::$_HOSTGRAPHITEPREFIX$\tGRAPHITEPOSTFIX::$_HOSTGRAPHITEPOSTFIX$\tMETRICTYPE::$_HOSTMETRICTYPE$ +host_perfdata_file_template=\tDATATYPE::HOSTPERFDATA\tTIMET::$TIMET$\tHOSTNAME::$HOSTNAME$\tHOSTPERFDATA::$HOSTPERFDATA$\tHOSTCHECKCOMMAND::$HOSTCHECKCOMMAND$\tHOSTSTATE::$HOSTSTATE$\tHOSTSTATETYPE::$HOSTSTATETYPE$\tGRAPHITEPREFIX::$_HOSTGRAPHITEPREFIX$\tGRAPHITEPOSTFIX::$_HOSTGRAPHITEPOSTFIX$\tMETRICTYPE::$_HOSTMETRICTYPE$ host_perfdata_file_mode=a host_perfdata_file_processing_interval=15 host_perfdata_file_processing_command=graphios_perf_host diff --git a/plugins/plugin_example.py b/plugins/plugin_example.py index 0467cac..1dd8df7 100644 --- a/plugins/plugin_example.py +++ b/plugins/plugin_example.py @@ -8,13 +8,28 @@ # It expects the output to be a list of metric paths and values and # Graphios will use the nag.TIMET timestamp to form the correct metric # +# For example: +# +# get_metrics() -> [ ( "stats.server1.cpu", .10 ) ] +# +# Would result in the following metric: +# +# stats.server1.cpu .10 +# +# Graphios will append the timestamp, only the path, and value pair must be +# returned +# def get_metrics(perfdata, nag): """ returns a [(, )] where each is the metric to send to carbon """ - path = "%s.%s.%s.%s" % (nag.GRAPHITEPREFIX, nag.HOSTNAME, nag.GRAPHITEPOSTFIX, nag.LABEL) - value = nag.VALUE + results = [] + for metric in perfdata.split(): + label = perfdata.split('=')[0] + path = "%s.%s.%s" % (nag.GRAPHITEPREFIX, nag.HOSTNAME, label) + value = nag.VALUE + results.append((path, value)) - return [ (path, value) ] + return results From 2b9dffa7debf77c79afeb5d8e1558b4df76564cc Mon Sep 17 00:00:00 2001 From: Connor Osborn Date: Wed, 1 Jul 2015 11:38:43 -0700 Subject: [PATCH 6/6] Clean up whitespace for flake8 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 16382fe..aa7aa12 100644 --- a/setup.py +++ b/setup.py @@ -91,8 +91,8 @@ def _post_install(): """ tries to find the nagios.cfg and insert graphios perf commands/cfg """ - lookin = ['/etc/nagios3/','/etc/nagios/', '/opt/nagios/', '/usr/local/nagios', - '/usr/nagios'] + lookin = ['/etc/nagios3/', '/etc/nagios/', '/opt/nagios/', + '/usr/local/nagios', '/usr/nagios'] nag_cfg = find_nagios_cfg(lookin) if nag_cfg is None: print("sorry I couldn't find the nagios.cfg file")