From be39855d7f6b4a5f597a9f8ba9bf9c1aab16f0ac Mon Sep 17 00:00:00 2001 From: Sergi Delgado Segura Date: Wed, 1 Apr 2020 10:56:59 +0200 Subject: [PATCH 01/18] Adds last_known_block to Watcher and stores last block on db on fresh bootstrap Watcher and Responder --- teos/responder.py | 1 + teos/watcher.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/teos/responder.py b/teos/responder.py index 526850f1..e1dd6950 100644 --- a/teos/responder.py +++ b/teos/responder.py @@ -268,6 +268,7 @@ def do_watch(self): # Distinguish fresh bootstraps from bootstraps from db if self.last_known_block is None: self.last_known_block = self.block_processor.get_best_block_hash() + self.db_manager.store_last_block_hash_responder(self.last_known_block) while True: block_hash = self.block_queue.get() diff --git a/teos/watcher.py b/teos/watcher.py index 33efa3cc..eefbd0f7 100644 --- a/teos/watcher.py +++ b/teos/watcher.py @@ -51,9 +51,10 @@ class Watcher: block_processor (:obj:`BlockProcessor `): a ``BlockProcessor`` instance to get block from bitcoind. responder (:obj:`Responder `): a ``Responder`` instance. - signing_key (:mod:`PrivateKey`): a private key used to sign accepted appointments. max_appointments (:obj:`int`): the maximum ammount of appointments accepted by the ``Watcher`` at the same time. expiry_delta (:obj:`int`): the additional time the ``Watcher`` will keep an expired appointment around. + signing_key (:mod:`PrivateKey`): a private key used to sign accepted appointments. + last_known_block (:obj:`str`): the last block known by the ``Watcher``. Raises: ValueError: if `teos_sk_file` is not found. @@ -70,6 +71,7 @@ def __init__(self, db_manager, block_processor, responder, sk_der, max_appointme self.max_appointments = max_appointments self.expiry_delta = expiry_delta self.signing_key = Cryptographer.load_private_key_der(sk_der) + self.last_known_block = db_manager.load_last_block_hash_responder() def awake(self): watcher_thread = Thread(target=self.do_watch, daemon=True) @@ -141,6 +143,11 @@ def do_watch(self): :obj:`Responder ` upon detecting a breach. """ + # Distinguish fresh bootstraps from bootstraps from db + if self.last_known_block is None: + self.last_known_block = self.block_processor.get_best_block_hash() + self.db_manager.store_last_block_hash_watcher(self.last_known_block) + while True: block_hash = self.block_queue.get() block = self.block_processor.get_block(block_hash) @@ -203,6 +210,7 @@ def do_watch(self): # Register the last processed block for the watcher self.db_manager.store_last_block_hash_watcher(block_hash) + self.last_known_block = block.get("hash") self.block_queue.task_done() def get_breaches(self, txids): From c3c8217728ba138a96931b99d377c9e8a2edba04 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 4 Apr 2020 13:47:14 -0400 Subject: [PATCH 02/18] Load teos logs into program --- monitor/monitor.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 monitor/monitor.py diff --git a/monitor/monitor.py b/monitor/monitor.py new file mode 100644 index 00000000..c773db4e --- /dev/null +++ b/monitor/monitor.py @@ -0,0 +1,39 @@ +import json +import os +from elasticsearch import Elasticsearch + +es = Elasticsearch() + +log_path = os.path.expanduser("~/.teos/teos.log") + +# Need to keep reading the file as it grows. +def load_logs(): + """ + Reads teos log as JSON. Reads initial file, then + + Returns: + """ + + # Load the initial log file. + with open(log_path, "r") as log_file: + log_data = log_file.readlines() + + try: + log_json = json.dumps(log_data) + except TypeError: + print("Unable to serialize logs to JSON.") + +# Keep parsing data from constantly updating logs. +# with open(log_path, "r") as log_file: +# for line in tail(log_file): +# try: +# log_data = json.loads(line) +# except ValueError: +# continue # Read next line + +def main(): + load_logs() + +if __name__ == "__main__": + main() + From 20b76029879f2f6af9184d022a40d54be25699b6 Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 8 Apr 2020 23:52:35 -0400 Subject: [PATCH 03/18] Index log data via Elasticsearch --- monitor/__init__.py | 0 monitor/log.py | 2 + monitor/monitor.py | 39 ------------------- monitor/monitord.py | 91 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 39 deletions(-) create mode 100644 monitor/__init__.py create mode 100644 monitor/log.py delete mode 100644 monitor/monitor.py create mode 100644 monitor/monitord.py diff --git a/monitor/__init__.py b/monitor/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/monitor/log.py b/monitor/log.py new file mode 100644 index 00000000..a58c8f6c --- /dev/null +++ b/monitor/log.py @@ -0,0 +1,2 @@ +LOG_PREFIX = "System monitor" + diff --git a/monitor/monitor.py b/monitor/monitor.py deleted file mode 100644 index c773db4e..00000000 --- a/monitor/monitor.py +++ /dev/null @@ -1,39 +0,0 @@ -import json -import os -from elasticsearch import Elasticsearch - -es = Elasticsearch() - -log_path = os.path.expanduser("~/.teos/teos.log") - -# Need to keep reading the file as it grows. -def load_logs(): - """ - Reads teos log as JSON. Reads initial file, then - - Returns: - """ - - # Load the initial log file. - with open(log_path, "r") as log_file: - log_data = log_file.readlines() - - try: - log_json = json.dumps(log_data) - except TypeError: - print("Unable to serialize logs to JSON.") - -# Keep parsing data from constantly updating logs. -# with open(log_path, "r") as log_file: -# for line in tail(log_file): -# try: -# log_data = json.loads(line) -# except ValueError: -# continue # Read next line - -def main(): - load_logs() - -if __name__ == "__main__": - main() - diff --git a/monitor/monitord.py b/monitor/monitord.py new file mode 100644 index 00000000..bdb5eff2 --- /dev/null +++ b/monitor/monitord.py @@ -0,0 +1,91 @@ +import json +import os +from elasticsearch import Elasticsearch, helpers +from elasticsearch.helpers.errors import BulkIndexError + +from log import LOG_PREFIX +from common.logger import Logger + +logger = Logger(actor="System Monitor", log_name_prefix=LOG_PREFIX) + + + +es = Elasticsearch() + +log_path = os.path.expanduser("~/.teos/teos.log") + + +# TODO: Logs are constantly being updated. Should we keep that data updated? +def load_logs(log_path): + """ + Reads teos log into a list. + + Returns: + :obj:`list`: The logs in the form of a list. + + Raises: + FileNotFoundError: If path doesn't correspond to an existing log file. + + """ + + # Load the initial log file. + with open(log_path, "r") as log_file: + log_data = log_file.readlines() + return log_data + + # TODO: Throw an error if the file is empty or if data isn't JSON. + + +def gen_log_data(log_data): + """ + Formats logs so it can be sent to Elasticsearch in bulk. + + Args: + log_data (:obj:`list`): The logs in list form. + + Yields: + :obj:`dict`: A dict conforming to the required format for sending data to elasticsearch in bulk. + """ + + for log in log_data: + yield { + "_index": "logs", + "_type": "document", + "doc": {"log": log}, + } + +def index_logs(log_data): + """ + Indexes logs in elasticsearch so they can be searched. + + Args: + logs (:obj:`str`): The logs in JSON form. + + Returns: + response (:obj:`tuple`): The first value of the tuple equals the number of the logs data was entered successfully. If there are errors the second value in the tuple includes the errors. + + Raises: + elasticsearch.helpers.errors.BulkIndexError: Returned by Elasticsearch if indexing log data fails. + + """ + + response = helpers.bulk(es, gen_log_data(log_data)) + + # The response is a tuple of two items: 1) The number of items successfully indexed. 2) Any errors returned. + + if response[0] == 0: + logger.error("Indexing logs in Elasticsearch error: ", e) + raise BulkIndexError() + + return response + + +def main(): + logger.info("Setting up the system monitor.") + + log_data = load_logs(log_path) + + index_logs(log_data) + +if __name__ == "__main__": + main() From 3879a90f37b973dc229f518f442d61a30c11354b Mon Sep 17 00:00:00 2001 From: Turtle Date: Wed, 8 Apr 2020 23:53:03 -0400 Subject: [PATCH 04/18] Test initial data gathering functions --- monitor/test_monitord.py | 50 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 monitor/test_monitord.py diff --git a/monitor/test_monitord.py b/monitor/test_monitord.py new file mode 100644 index 00000000..d557912e --- /dev/null +++ b/monitor/test_monitord.py @@ -0,0 +1,50 @@ +import json +import os +import pytest + +from monitor.monitord import load_logs, gen_log_data, index_logs + +test_log_data = [ + {"locator": "bab905e8279395b663bf2feca5213dc5", "message": "New appointment accepted", "time": "01/04/2020 15:53:15"}, + {"message": "Shutting down TEOS", "time": "01/04/2020 15:53:31"} +] + + + +def test_load_logs(): + # Create a temporary file with some test logs inside. + with open("test_log_file", "w") as f: + for log in test_log_data: + f.write("{}\n".format(log)) + + # Make sure load_logs function returns the logs in list form. + log_data = load_logs("test_log_file") + assert len(log_data) == 2 + + # Delete the temporary file. + os.remove("test_log_file") + + +def test_load_logs_err(): + # If file doesn't exist, load_logs should throw an error. + with pytest.raises(FileNotFoundError): + load_logs("nonexistent_log_file") + + +# NOTE/TODO: Elasticsearch needs to be running for this test to work. +def test_index_logs(): + json_logs = [] + for log in test_log_data: + json_logs.append(json.dumps(log)) + + response = index_logs(json_logs) + + assert type(response) is tuple + assert len(response) == 2 + assert response[0] == 2 + + # TODO: Delete logs from elasticsearch that were indexed + + +# TODO: Test that a invalid data sent to index_logs is handled correctly. + From 6a7ba120199d93d80c39e5289d98858ba8c1229a Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 11 Apr 2020 14:10:05 -0400 Subject: [PATCH 05/18] Build out Elasticsearch functionality --- monitor/monitord.py | 95 +++++++++++++++++++++++++++++++++++----- monitor/test_monitord.py | 6 ++- 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/monitor/monitord.py b/monitor/monitord.py index bdb5eff2..174d4520 100644 --- a/monitor/monitord.py +++ b/monitor/monitord.py @@ -20,8 +20,11 @@ def load_logs(log_path): """ Reads teos log into a list. + Args: + log_path (:obj:`str`): The path to the log file. + Returns: - :obj:`list`: The logs in the form of a list. + :obj:`list`: A list of logs in dict form. Raises: FileNotFoundError: If path doesn't correspond to an existing log file. @@ -29,11 +32,15 @@ def load_logs(log_path): """ # Load the initial log file. + logs = [] with open(log_path, "r") as log_file: - log_data = log_file.readlines() - return log_data + for log in log_file: + log_data = json.loads(log.strip()) + logs.append(log_data) + + return logs - # TODO: Throw an error if the file is empty or if data isn't JSON. + # TODO: Throw an error if the file is empty or if data isn't JSON-y. def gen_log_data(log_data): @@ -41,25 +48,29 @@ def gen_log_data(log_data): Formats logs so it can be sent to Elasticsearch in bulk. Args: - log_data (:obj:`list`): The logs in list form. + log_data (:obj:`list`): A list of logs in dict form. Yields: :obj:`dict`: A dict conforming to the required format for sending data to elasticsearch in bulk. """ for log in log_data: + # We don't need to include errors (which had problems mapping anyway) + if 'error' in log: + continue yield { "_index": "logs", "_type": "document", - "doc": {"log": log}, + "doc": log } + def index_logs(log_data): """ Indexes logs in elasticsearch so they can be searched. Args: - logs (:obj:`str`): The logs in JSON form. + logs (:obj:`list`): A list of logs in dict form. Returns: response (:obj:`tuple`): The first value of the tuple equals the number of the logs data was entered successfully. If there are errors the second value in the tuple includes the errors. @@ -74,18 +85,78 @@ def index_logs(log_data): # The response is a tuple of two items: 1) The number of items successfully indexed. 2) Any errors returned. if response[0] == 0: - logger.error("Indexing logs in Elasticsearch error: ", e) - raise BulkIndexError() + logger.error("None of the logs were indexed. Log data might be in the wrong form.") return response +def search_logs(field, keyword, index): + """ + Searches Elasticsearch for data with a certain field and keyword. + + Args: + field (:obj:`str`): The search field. + keyword (:obj:`str`): The search keyword. + index (:obj:`str`): The index in Elasticsearch to search through. + + Returns: + :obj:`dict`: A dict describing the results, including the first 10 docs matching the search words. + """ + + body = { + "query": {"match": {"doc.{}".format(field): keyword}} + } + results = es.search(body, index) + + return results + + +def get_all_logs(): + """ + Retrieves all logs in the logs index of Elasticsearch. + + Returns: + :obj:`dict`: A dict describing the results, including the first 10 docs. + """ + + body = { + "query": { "match_all": {} } + } + results = es.search(body, "logs") + + results = json.dumps(results, indent=4) + + return results + + +def delete_all_by_index(index): + """ + Deletes all logs in the chosen index of Elasticsearch. + + Args: + index (:obj:`str`): The index in Elasticsearch. + + Returns: + :obj:`dict`: A dict describing how many items were deleted and including any deletion failures. + + """ + + body = { + "query": { "match_all": {} } + } + results = es.delete_by_query(index, body) + + return results + + def main(): logger.info("Setting up the system monitor.") - log_data = load_logs(log_path) - - index_logs(log_data) + # log_data = load_logs(log_path) + # index_logs(log_data) + # search_logs("message", ["logs"]) + # get_all_logs() + # delete_all_by_index("logs") if __name__ == "__main__": main() diff --git a/monitor/test_monitord.py b/monitor/test_monitord.py index d557912e..efb4424b 100644 --- a/monitor/test_monitord.py +++ b/monitor/test_monitord.py @@ -15,7 +15,7 @@ def test_load_logs(): # Create a temporary file with some test logs inside. with open("test_log_file", "w") as f: for log in test_log_data: - f.write("{}\n".format(log)) + f.write(json.dumps(log) + "\n") # Make sure load_logs function returns the logs in list form. log_data = load_logs("test_log_file") @@ -30,12 +30,14 @@ def test_load_logs_err(): with pytest.raises(FileNotFoundError): load_logs("nonexistent_log_file") + # TODO: Test if it raises an error if the file is empty. + # NOTE/TODO: Elasticsearch needs to be running for this test to work. def test_index_logs(): json_logs = [] for log in test_log_data: - json_logs.append(json.dumps(log)) + json_logs.append(log) response = index_logs(json_logs) From 4933b82e8c814d8583c48f23e71f070e81ac995e Mon Sep 17 00:00:00 2001 From: Turtle Date: Sun, 12 Apr 2020 17:36:21 -0400 Subject: [PATCH 06/18] Refactor into OO --- monitor/log.py | 2 - monitor/monitor.py | 16 ++ monitor/monitord.py | 162 ------------------ monitor/searcher.py | 161 +++++++++++++++++ monitor/test/__init__.py | 0 .../test_searcher.py} | 22 ++- 6 files changed, 191 insertions(+), 172 deletions(-) delete mode 100644 monitor/log.py create mode 100644 monitor/monitor.py delete mode 100644 monitor/monitord.py create mode 100644 monitor/searcher.py create mode 100644 monitor/test/__init__.py rename monitor/{test_monitord.py => test/test_searcher.py} (75%) diff --git a/monitor/log.py b/monitor/log.py deleted file mode 100644 index a58c8f6c..00000000 --- a/monitor/log.py +++ /dev/null @@ -1,2 +0,0 @@ -LOG_PREFIX = "System monitor" - diff --git a/monitor/monitor.py b/monitor/monitor.py new file mode 100644 index 00000000..a16cab76 --- /dev/null +++ b/monitor/monitor.py @@ -0,0 +1,16 @@ +from monitor.searcher import Searcher + +from common.logger import Logger + +logger = Logger(actor="System Monitor Main", log_name_prefix=LOG_PREFIX) + +def main(): + logger.info("Setting up the system monitor.") + + # Create and start searcher. + searcher = Searcher() + searcher.start() + +if __name__ == "__main__": + main() + diff --git a/monitor/monitord.py b/monitor/monitord.py deleted file mode 100644 index 174d4520..00000000 --- a/monitor/monitord.py +++ /dev/null @@ -1,162 +0,0 @@ -import json -import os -from elasticsearch import Elasticsearch, helpers -from elasticsearch.helpers.errors import BulkIndexError - -from log import LOG_PREFIX -from common.logger import Logger - -logger = Logger(actor="System Monitor", log_name_prefix=LOG_PREFIX) - - - -es = Elasticsearch() - -log_path = os.path.expanduser("~/.teos/teos.log") - - -# TODO: Logs are constantly being updated. Should we keep that data updated? -def load_logs(log_path): - """ - Reads teos log into a list. - - Args: - log_path (:obj:`str`): The path to the log file. - - Returns: - :obj:`list`: A list of logs in dict form. - - Raises: - FileNotFoundError: If path doesn't correspond to an existing log file. - - """ - - # Load the initial log file. - logs = [] - with open(log_path, "r") as log_file: - for log in log_file: - log_data = json.loads(log.strip()) - logs.append(log_data) - - return logs - - # TODO: Throw an error if the file is empty or if data isn't JSON-y. - - -def gen_log_data(log_data): - """ - Formats logs so it can be sent to Elasticsearch in bulk. - - Args: - log_data (:obj:`list`): A list of logs in dict form. - - Yields: - :obj:`dict`: A dict conforming to the required format for sending data to elasticsearch in bulk. - """ - - for log in log_data: - # We don't need to include errors (which had problems mapping anyway) - if 'error' in log: - continue - yield { - "_index": "logs", - "_type": "document", - "doc": log - } - - -def index_logs(log_data): - """ - Indexes logs in elasticsearch so they can be searched. - - Args: - logs (:obj:`list`): A list of logs in dict form. - - Returns: - response (:obj:`tuple`): The first value of the tuple equals the number of the logs data was entered successfully. If there are errors the second value in the tuple includes the errors. - - Raises: - elasticsearch.helpers.errors.BulkIndexError: Returned by Elasticsearch if indexing log data fails. - - """ - - response = helpers.bulk(es, gen_log_data(log_data)) - - # The response is a tuple of two items: 1) The number of items successfully indexed. 2) Any errors returned. - - if response[0] == 0: - logger.error("None of the logs were indexed. Log data might be in the wrong form.") - - return response - - -def search_logs(field, keyword, index): - """ - Searches Elasticsearch for data with a certain field and keyword. - - Args: - field (:obj:`str`): The search field. - keyword (:obj:`str`): The search keyword. - index (:obj:`str`): The index in Elasticsearch to search through. - - Returns: - :obj:`dict`: A dict describing the results, including the first 10 docs matching the search words. - """ - - body = { - "query": {"match": {"doc.{}".format(field): keyword}} - } - results = es.search(body, index) - - return results - - -def get_all_logs(): - """ - Retrieves all logs in the logs index of Elasticsearch. - - Returns: - :obj:`dict`: A dict describing the results, including the first 10 docs. - """ - - body = { - "query": { "match_all": {} } - } - results = es.search(body, "logs") - - results = json.dumps(results, indent=4) - - return results - - -def delete_all_by_index(index): - """ - Deletes all logs in the chosen index of Elasticsearch. - - Args: - index (:obj:`str`): The index in Elasticsearch. - - Returns: - :obj:`dict`: A dict describing how many items were deleted and including any deletion failures. - - """ - - body = { - "query": { "match_all": {} } - } - results = es.delete_by_query(index, body) - - return results - - -def main(): - logger.info("Setting up the system monitor.") - - # log_data = load_logs(log_path) - # index_logs(log_data) - # search_logs("message", ["logs"]) - # get_all_logs() - # delete_all_by_index("logs") - -if __name__ == "__main__": - main() diff --git a/monitor/searcher.py b/monitor/searcher.py new file mode 100644 index 00000000..fe9a6ea2 --- /dev/null +++ b/monitor/searcher.py @@ -0,0 +1,161 @@ +import json +import os +from elasticsearch import Elasticsearch, helpers +from elasticsearch.helpers.errors import BulkIndexError + +from common.logger import Logger + +LOG_PREFIX = "System Monitor" +logger = Logger(actor="Searcher", log_name_prefix=LOG_PREFIX) + + +class Searcher: + def __init__(self): + self.es = Elasticsearch() + # TODO: Pass the path through as a config option. + self.log_path = os.path.expanduser("~/.teos/teos.log") + + def start(self): + # Pull the watchtower logs into Elasticsearch. + log_data = self.load_logs(log_path) + self.index_logs(log_data) + + # Search for the data we need to visualize a graph. + + # self.search_logs("message", ["logs"]) + # self.get_all_logs() + # self.delete_all_by_index("logs") + + # TODO: Logs are constantly being updated. Keep that data updated + def load_logs(self, log_path): + """ + Reads teos log into a list. + + Args: + log_path (:obj:`str`): The path to the log file. + + Returns: + :obj:`list`: A list of logs in dict form. + + Raises: + FileNotFoundError: If path doesn't correspond to an existing log file. + + """ + + # Load the initial log file. + logs = [] + with open(log_path, "r") as log_file: + for log in log_file: + log_data = json.loads(log.strip()) + logs.append(log_data) + + return logs + + # TODO: Throw an error if the file is empty or if data isn't JSON-y. + + + @staticmethod + def gen_log_data(log_data): + """ + Formats logs so it can be sent to Elasticsearch in bulk. + + Args: + log_data (:obj:`list`): A list of logs in dict form. + + Yields: + :obj:`dict`: A dict conforming to the required format for sending data to elasticsearch in bulk. + """ + + for log in log_data: + # We don't need to include errors (which had problems mapping anyway) + if 'error' in log: + continue + yield { + "_index": "logs", + "_type": "document", + "doc": log + } + + + def index_logs(self, log_data): + """ + Indexes logs in elasticsearch so they can be searched. + + Args: + logs (:obj:`list`): A list of logs in dict form. + + Returns: + response (:obj:`tuple`): The first value of the tuple equals the number of the logs data was entered successfully. If there are errors the second value in the tuple includes the errors. + + Raises: + elasticsearch.helpers.errors.BulkIndexError: Returned by Elasticsearch if indexing log data fails. + + """ + + response = helpers.bulk(self.es, self.gen_log_data(log_data)) + # print("response: ", response) + + # The response is a tuple of two items: 1) The number of items successfully indexed. 2) Any errors returned. + if (response[0] <= 0): + logger.error("None of the logs were indexed. Log data might be in the wrong form.") + + return response + + + def search_logs(self, field, keyword, index): + """ + Searches Elasticsearch for data with a certain field and keyword. + + Args: + field (:obj:`str`): The search field. + keyword (:obj:`str`): The search keyword. + index (:obj:`str`): The index in Elasticsearch to search through. + + Returns: + :obj:`dict`: A dict describing the results, including the first 10 docs matching the search words. + """ + + body = { + "query": {"match": {"doc.{}".format(field): keyword}} + } + results = self.es.search(body, index) + + return results + + + def get_all_logs(self): + """ + Retrieves all logs in the logs index of Elasticsearch. + + Returns: + :obj:`dict`: A dict describing the results, including the first 10 docs. + """ + + body = { + "query": { "match_all": {} } + } + results = self.es.search(body, "logs") + + results = json.dumps(results, indent=4) + + return results + + + def delete_all_by_index(self, index): + """ + Deletes all logs in the chosen index of Elasticsearch. + + Args: + index (:obj:`str`): The index in Elasticsearch. + + Returns: + :obj:`dict`: A dict describing how many items were deleted and including any deletion failures. + + """ + + body = { + "query": { "match_all": {} } + } + results = self.es.delete_by_query(index, body) + + return results diff --git a/monitor/test/__init__.py b/monitor/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/monitor/test_monitord.py b/monitor/test/test_searcher.py similarity index 75% rename from monitor/test_monitord.py rename to monitor/test/test_searcher.py index efb4424b..cc41c582 100644 --- a/monitor/test_monitord.py +++ b/monitor/test/test_searcher.py @@ -2,7 +2,7 @@ import os import pytest -from monitor.monitord import load_logs, gen_log_data, index_logs +from monitor.searcher import Searcher test_log_data = [ {"locator": "bab905e8279395b663bf2feca5213dc5", "message": "New appointment accepted", "time": "01/04/2020 15:53:15"}, @@ -11,35 +11,42 @@ -def test_load_logs(): +@pytest.fixture(scope="module") +def searcher(): + searcher = Searcher() + + return searcher + + +def test_load_logs(searcher): # Create a temporary file with some test logs inside. with open("test_log_file", "w") as f: for log in test_log_data: f.write(json.dumps(log) + "\n") # Make sure load_logs function returns the logs in list form. - log_data = load_logs("test_log_file") + log_data = searcher.load_logs("test_log_file") assert len(log_data) == 2 # Delete the temporary file. os.remove("test_log_file") -def test_load_logs_err(): +def test_load_logs_err(searcher): # If file doesn't exist, load_logs should throw an error. with pytest.raises(FileNotFoundError): - load_logs("nonexistent_log_file") + searcher.load_logs("nonexistent_log_file") # TODO: Test if it raises an error if the file is empty. # NOTE/TODO: Elasticsearch needs to be running for this test to work. -def test_index_logs(): +def test_index_logs(searcher): json_logs = [] for log in test_log_data: json_logs.append(log) - response = index_logs(json_logs) + response = searcher.index_logs(json_logs) assert type(response) is tuple assert len(response) == 2 @@ -49,4 +56,3 @@ def test_index_logs(): # TODO: Test that a invalid data sent to index_logs is handled correctly. - From 51e6ad74811ff3465de0c21b3254913b84fd44c6 Mon Sep 17 00:00:00 2001 From: Turtle Date: Tue, 28 Apr 2020 21:37:13 -0400 Subject: [PATCH 07/18] Create index mapping for doc.time date type --- .gitignore | 2 + monitor/{monitor.py => monitor_start.py} | 3 +- monitor/searcher.py | 52 +++++++++++++++++------- 3 files changed, 41 insertions(+), 16 deletions(-) rename monitor/{monitor.py => monitor_start.py} (77%) diff --git a/.gitignore b/.gitignore index 64b00b01..25db94ea 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ htmlcov docs/ .teos .teos_cli + +/monitor/monitor.conf diff --git a/monitor/monitor.py b/monitor/monitor_start.py similarity index 77% rename from monitor/monitor.py rename to monitor/monitor_start.py index a16cab76..615f7b51 100644 --- a/monitor/monitor.py +++ b/monitor/monitor_start.py @@ -2,13 +2,14 @@ from common.logger import Logger +LOG_PREFIX = "Main" logger = Logger(actor="System Monitor Main", log_name_prefix=LOG_PREFIX) def main(): logger.info("Setting up the system monitor.") # Create and start searcher. - searcher = Searcher() + searcher = Searcher(None, None, CLOUD_ID, AUTH_USER, AUTH_PW) searcher.start() if __name__ == "__main__": diff --git a/monitor/searcher.py b/monitor/searcher.py index fe9a6ea2..79e22a85 100644 --- a/monitor/searcher.py +++ b/monitor/searcher.py @@ -1,6 +1,7 @@ import json import os from elasticsearch import Elasticsearch, helpers +from elasticsearch.client import IndicesClient from elasticsearch.helpers.errors import BulkIndexError from common.logger import Logger @@ -10,21 +11,45 @@ class Searcher: - def __init__(self): - self.es = Elasticsearch() + def __init__(self, host, port, cloud_id=None, auth_user=None, auth_pw=None): + self.es_host = host + self.es_port = port + self.es_cloud_id = cloud_id + self.es_auth_user = auth_user + self.es_auth_pw = auth_pw + self.es = Elasticsearch( + cloud_id=self.es_cloud_id, + http_auth=(self.es_auth_user, self.es_auth_pw), + ) + self.index_client = IndicesClient(self.es) # TODO: Pass the path through as a config option. - self.log_path = os.path.expanduser("~/.teos/teos.log") + self.log_path = os.path.expanduser("~/.teos/teos_test.log") def start(self): # Pull the watchtower logs into Elasticsearch. - log_data = self.load_logs(log_path) - self.index_logs(log_data) + # self.index_client.delete("logs") + # self.create_index("logs") + # log_data = self.load_logs(self.log_path) + # self.index_logs(log_data) # Search for the data we need to visualize a graph. # self.search_logs("message", ["logs"]) - # self.get_all_logs() + self.get_all_logs() # self.delete_all_by_index("logs") + + def create_index(self, name): + body = { + "mappings": { + "properties": { + "doc.time": { + "type": "date", + "format": "strict_date_optional_time||dd/MM/yyyy HH:mm:ss" + } + } + } + } + self.index_client.create(name, body) # TODO: Logs are constantly being updated. Keep that data updated def load_logs(self, log_path): @@ -48,12 +73,11 @@ def load_logs(self, log_path): for log in log_file: log_data = json.loads(log.strip()) logs.append(log_data) - + return logs # TODO: Throw an error if the file is empty or if data isn't JSON-y. - @staticmethod def gen_log_data(log_data): """ @@ -68,11 +92,11 @@ def gen_log_data(log_data): for log in log_data: # We don't need to include errors (which had problems mapping anyway) - if 'error' in log: - continue + # if 'error' in log: + # continue yield { "_index": "logs", - "_type": "document", + # "_type": "document", "doc": log } @@ -93,15 +117,13 @@ def index_logs(self, log_data): """ response = helpers.bulk(self.es, self.gen_log_data(log_data)) - # print("response: ", response) # The response is a tuple of two items: 1) The number of items successfully indexed. 2) Any errors returned. if (response[0] <= 0): logger.error("None of the logs were indexed. Log data might be in the wrong form.") - + return response - - + def search_logs(self, field, keyword, index): """ Searches Elasticsearch for data with a certain field and keyword. From 212aef5cc43a1aad7ab456d10f3d684534ae3d8a Mon Sep 17 00:00:00 2001 From: Turtle Date: Sat, 2 May 2020 11:34:41 -0400 Subject: [PATCH 08/18] Update searcher docs --- monitor/searcher.py | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/monitor/searcher.py b/monitor/searcher.py index 79e22a85..469cc811 100644 --- a/monitor/searcher.py +++ b/monitor/searcher.py @@ -11,6 +11,28 @@ class Searcher: + """ + The :class:`Searcher` is in charge of the monitor's Elasticsearch functionality for loading and searching through data. + + Args: + host (:obj:`str`): The host Elasticsearch is running on. + port (:obj:`int`): The port Elasticsearch is runnning on. + cloud_id (:obj:`str`): Elasticsearch cloud id, if Elasticsearch Cloud is being used. + auth_user (:obj:`str`): Elasticsearch Cloud username, if Elasticsearch Cloud is being used. + auth_pw (:obj:`str`): Elasticsearch Cloud password, if Elasticsearch Cloud is being used. + + Attributes: + host (:obj:`str`): The host Elasticsearch is running on. + port (:obj:`int`): The port Elasticsearch is runnning on. + cloud_id (:obj:`str`): Elasticsearch cloud id, if Elasticsearch Cloud is being used. + auth_user (:obj:`str`): Elasticsearch Cloud username, if Elasticsearch Cloud is being used. + auth_pw (:obj:`str`): Elasticsearch Cloud password, if Elasticsearch Cloud is being used. + es (:obj:`Elasticsearch `): The Elasticsearch client for searching for data to be visualized. + index_client (:obj:`IndicesClient `): The index client where log data is stored. + log_path (:obj:`str`): The path to the log file where log file will be pulled from and analyzed by ES. + + """ + def __init__(self, host, port, cloud_id=None, auth_user=None, auth_pw=None): self.es_host = host self.es_port = port @@ -26,9 +48,11 @@ def __init__(self, host, port, cloud_id=None, auth_user=None, auth_pw=None): self.log_path = os.path.expanduser("~/.teos/teos_test.log") def start(self): + """Starts Elasticsearch and compiles data to be visualized in Kibana""" + # Pull the watchtower logs into Elasticsearch. # self.index_client.delete("logs") - # self.create_index("logs") + # self.create_log_index("logs") # log_data = self.load_logs(self.log_path) # self.index_logs(log_data) @@ -38,7 +62,15 @@ def start(self): self.get_all_logs() # self.delete_all_by_index("logs") - def create_index(self, name): + def create_log_index(self, index): + """ + Create index with a particular mapping. + + Args: + index (:obj:`str`): Index the mapping is in. + + """ + body = { "mappings": { "properties": { @@ -49,7 +81,7 @@ def create_index(self, name): } } } - self.index_client.create(name, body) + self.index_client.create(index, body) # TODO: Logs are constantly being updated. Keep that data updated def load_logs(self, log_path): From 2ed0cc71d95af36ae211287a7f2bc6c3eec1379f Mon Sep 17 00:00:00 2001 From: Turtle Date: Fri, 15 May 2020 12:45:42 -0400 Subject: [PATCH 09/18] Load config options --- monitor/__init__.py | 14 +++++ monitor/monitor_start.py | 31 +++++++++-- monitor/searcher.py | 112 ++++++++++++++++++++++++++++----------- 3 files changed, 122 insertions(+), 35 deletions(-) diff --git a/monitor/__init__.py b/monitor/__init__.py index e69de29b..6c0ec59d 100644 --- a/monitor/__init__.py +++ b/monitor/__init__.py @@ -0,0 +1,14 @@ +import os + +MONITOR_DIR = os.getcwd() + "/monitor/" +MONITOR_CONF = "monitor.conf" + +MONITOR_DEFAULT_CONF = { + "ES_HOST": {"value": "", "type": str}, + "ES_PORT": {"value": 9200, "type": int}, + "CLOUD_ID": {"value": "", "type": str}, + "AUTH_USER": {"value": "user", "type": str}, + "AUTH_PW": {"value": "password", "type": str}, + "KIBANA_HOST": {"value": "localhost", "type": str}, + "KIBANA_PORT": {"value": "9243", "type": int}, +} diff --git a/monitor/monitor_start.py b/monitor/monitor_start.py index 615f7b51..a1040b81 100644 --- a/monitor/monitor_start.py +++ b/monitor/monitor_start.py @@ -1,17 +1,42 @@ +from monitor import MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF from monitor.searcher import Searcher +from common.config_loader import ConfigLoader from common.logger import Logger +from teos import DATA_DIR, DEFAULT_CONF, CONF_FILE_NAME + LOG_PREFIX = "Main" logger = Logger(actor="System Monitor Main", log_name_prefix=LOG_PREFIX) -def main(): +def main(command_line_conf): logger.info("Setting up the system monitor.") + # Pull in Teos's config file to retrieve some of the data we need. + conf_loader = ConfigLoader(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF, command_line_conf) + conf = conf_loader.build_config() + + max_users = conf.get("DEFAULT_SLOTS") + api_host = conf.get("API_BIND") + api_port = conf.get("API_PORT") + log_file = conf.get("LOG_FILE") + + mon_conf_loader = ConfigLoader(MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF, command_line_conf) + mon_conf = mon_conf_loader.build_config() + + es_host = mon_conf.get("ES_HOST") + es_port = mon_conf.get("ES_PORT") + cloud_id = mon_conf.get("CLOUD_ID") + auth_user = mon_conf.get("AUTH_USER") + auth_pw = mon_conf.get("AUTH_PW") + # Create and start searcher. - searcher = Searcher(None, None, CLOUD_ID, AUTH_USER, AUTH_PW) + searcher = Searcher(es_host, es_port, api_host, api_port, DATA_DIR, log_file, cloud_id, auth_user, auth_pw) searcher.start() if __name__ == "__main__": - main() + # TODO: + command_line_conf = {} + + main(command_line_conf) diff --git a/monitor/searcher.py b/monitor/searcher.py index 469cc811..4b6442f6 100644 --- a/monitor/searcher.py +++ b/monitor/searcher.py @@ -1,9 +1,12 @@ import json import os +import time + from elasticsearch import Elasticsearch, helpers from elasticsearch.client import IndicesClient from elasticsearch.helpers.errors import BulkIndexError +from cli import teos_cli from common.logger import Logger LOG_PREFIX = "System Monitor" @@ -33,36 +36,42 @@ class Searcher: """ - def __init__(self, host, port, cloud_id=None, auth_user=None, auth_pw=None): - self.es_host = host - self.es_port = port + def __init__(self, es_host, es_port, api_host, api_port, teos_dir, log_file, cloud_id=None, auth_user=None, auth_pw=None): + self.es_host = es_host + self.es_port = es_port self.es_cloud_id = cloud_id self.es_auth_user = auth_user self.es_auth_pw = auth_pw - self.es = Elasticsearch( - cloud_id=self.es_cloud_id, - http_auth=(self.es_auth_user, self.es_auth_pw), - ) + if self.es_host is not "": + self.es = Elasticsearch([ + {'host': self.es_host, 'port': self.es_port} + ]) + elif cloud_id is not "": + self.es = Elasticsearch( + cloud_id=self.es_cloud_id, + http_auth=(self.es_auth_user, self.es_auth_pw), + ) self.index_client = IndicesClient(self.es) - # TODO: Pass the path through as a config option. - self.log_path = os.path.expanduser("~/.teos/teos_test.log") + self.log_path = "{}/{}".format(teos_dir, log_file) + self.api_host = api_host + self.api_port = api_port def start(self): """Starts Elasticsearch and compiles data to be visualized in Kibana""" + # self.delete_index("logs") + # Pull the watchtower logs into Elasticsearch. - # self.index_client.delete("logs") - # self.create_log_index("logs") - # log_data = self.load_logs(self.log_path) - # self.index_logs(log_data) + #self.create_index("logs") + #log_data = self.load_logs(self.log_path) + #self.index_logs_bulk(log_data) # Search for the data we need to visualize a graph. + # self.load_and_index_other_data() # self.search_logs("message", ["logs"]) - self.get_all_logs() - # self.delete_all_by_index("logs") - def create_log_index(self, index): + def create_index(self, index): """ Create index with a particular mapping. @@ -76,13 +85,17 @@ def create_log_index(self, index): "properties": { "doc.time": { "type": "date", - "format": "strict_date_optional_time||dd/MM/yyyy HH:mm:ss" + "format": "epoch_second||strict_date_optional_time||dd/MM/yyyy HH:mm:ss" + }, + "doc.error.code": { + "type": "integer" } } } } - self.index_client.create(index, body) - + + resp = self.index_client.create(index, body) + # TODO: Logs are constantly being updated. Keep that data updated def load_logs(self, log_path): """ @@ -109,9 +122,32 @@ def load_logs(self, log_path): return logs # TODO: Throw an error if the file is empty or if data isn't JSON-y. - + + def load_and_index_other_data(self): + """ + Loads and indexes the rest of the data into Elasticsearch that we'll need to visualize using Kibana. + + """ + + # Grab # of appointments in watcher and responder + num_appts = self.get_num_appointments() + watcher_appts = num_appts[0] + responder_appts = num_appts[1] + + # index current number of appointments in watcher and responder + self.index_item("logs", "watcher_appts", watcher_appts) + self.index_item("logs", "responder_appts", responder_appts) + + def index_item(self, index, field, value): + body = { + field: value, + "doc.time": time.time() + } + + resp = self.es.index(index, body) + @staticmethod - def gen_log_data(log_data): + def gen_data(index, data): """ Formats logs so it can be sent to Elasticsearch in bulk. @@ -122,18 +158,13 @@ def gen_log_data(log_data): :obj:`dict`: A dict conforming to the required format for sending data to elasticsearch in bulk. """ - for log in log_data: - # We don't need to include errors (which had problems mapping anyway) - # if 'error' in log: - # continue + for log in data: yield { - "_index": "logs", - # "_type": "document", + "_index": index, "doc": log } - - def index_logs(self, log_data): + def index_data_bulk(self, index, data): """ Indexes logs in elasticsearch so they can be searched. @@ -148,7 +179,7 @@ def index_logs(self, log_data): """ - response = helpers.bulk(self.es, self.gen_log_data(log_data)) + response = helpers.bulk(self.es, self.gen_data(index, data)) # The response is a tuple of two items: 1) The number of items successfully indexed. 2) Any errors returned. if (response[0] <= 0): @@ -156,6 +187,18 @@ def index_logs(self, log_data): return response + def get_num_appointments(self): + teos_url = "http://{}:{}".format(self.api_host, self.api_port) + + resp = teos_cli.get_all_appointments(teos_url) + + response = json.loads(resp) + + watcher_appts = len(response.get("watcher_appointments")) + responder_appts = len(response.get("responder_trackers")) + + return [watcher_appts, responder_appts] + def search_logs(self, field, keyword, index): """ Searches Elasticsearch for data with a certain field and keyword. @@ -191,10 +234,15 @@ def get_all_logs(self): results = self.es.search(body, "logs") results = json.dumps(results, indent=4) - + return results + - + def delete_index(self, index): + + results = self.index_client.delete(index) + + def delete_all_by_index(self, index): """ Deletes all logs in the chosen index of Elasticsearch. From 485f054324418ea7f7f2f42a2190998686dcb518 Mon Sep 17 00:00:00 2001 From: Turtle Date: Fri, 15 May 2020 12:49:22 -0400 Subject: [PATCH 10/18] Update tests --- monitor/test/test_searcher.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/monitor/test/test_searcher.py b/monitor/test/test_searcher.py index cc41c582..853c8976 100644 --- a/monitor/test/test_searcher.py +++ b/monitor/test/test_searcher.py @@ -2,18 +2,40 @@ import os import pytest +from monitor import MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF from monitor.searcher import Searcher +from common.config_loader import ConfigLoader + +from teos import DATA_DIR, DEFAULT_CONF, CONF_FILE_NAME + + test_log_data = [ {"locator": "bab905e8279395b663bf2feca5213dc5", "message": "New appointment accepted", "time": "01/04/2020 15:53:15"}, {"message": "Shutting down TEOS", "time": "01/04/2020 15:53:31"} ] +conf_loader = ConfigLoader(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF, {}) +conf = conf_loader.build_config() + +api_host = conf.get("API_BIND") +api_port = conf.get("API_PORT") +log_file = conf.get("LOG_FILE") + +mon_conf_loader = ConfigLoader(MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF, {}) +mon_conf = mon_conf_loader.build_config() + +es_host = mon_conf.get("ES_HOST") +es_port = mon_conf.get("ES_PORT") +cloud_id = mon_conf.get("CLOUD_ID") +auth_user = mon_conf.get("AUTH_USER") +auth_pw = mon_conf.get("AUTH_PW") + @pytest.fixture(scope="module") def searcher(): - searcher = Searcher() + searcher = Searcher(es_host, es_port, api_host, api_port, DATA_DIR, log_file, cloud_id, auth_user, auth_pw) return searcher @@ -41,18 +63,19 @@ def test_load_logs_err(searcher): # NOTE/TODO: Elasticsearch needs to be running for this test to work. -def test_index_logs(searcher): +def test_index_data_bulk(searcher): json_logs = [] for log in test_log_data: json_logs.append(log) - response = searcher.index_logs(json_logs) + response = searcher.index_data_bulk("test-logs", json_logs) assert type(response) is tuple assert len(response) == 2 assert response[0] == 2 - # TODO: Delete logs from elasticsearch that were indexed + # Delete test logs from elasticsearch that were indexed. + searcher.delete_index("test-logs") # TODO: Test that a invalid data sent to index_logs is handled correctly. From 8d9602a6a8fadf73ffa3d8fabc15672cd3f36530 Mon Sep 17 00:00:00 2001 From: Turtle Date: Mon, 18 May 2020 23:47:36 -0400 Subject: [PATCH 11/18] Small searcher changes --- monitor/searcher.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/monitor/searcher.py b/monitor/searcher.py index 4b6442f6..e3701796 100644 --- a/monitor/searcher.py +++ b/monitor/searcher.py @@ -52,7 +52,7 @@ def __init__(self, es_host, es_port, api_host, api_port, teos_dir, log_file, clo http_auth=(self.es_auth_user, self.es_auth_pw), ) self.index_client = IndicesClient(self.es) - self.log_path = "{}/{}".format(teos_dir, log_file) + self.log_path = log_file self.api_host = api_host self.api_port = api_port @@ -62,9 +62,9 @@ def start(self): # self.delete_index("logs") # Pull the watchtower logs into Elasticsearch. - #self.create_index("logs") + # self.create_index("logs") #log_data = self.load_logs(self.log_path) - #self.index_logs_bulk(log_data) + #self.index_data_bulk("logs", log_data) # Search for the data we need to visualize a graph. # self.load_and_index_other_data() @@ -89,7 +89,13 @@ def create_index(self, index): }, "doc.error.code": { "type": "integer" - } + }, + "doc.watcher_appts": { + "type": "integer" + }, + "doc.responder_appts": { + "type": "integer" + } } } } @@ -140,7 +146,7 @@ def load_and_index_other_data(self): def index_item(self, index, field, value): body = { - field: value, + "doc.{}".format(field): value, "doc.time": time.time() } From 9754e10e63d93d1b7692a288a48a58094442497b Mon Sep 17 00:00:00 2001 From: Turtle Date: Mon, 18 May 2020 23:53:02 -0400 Subject: [PATCH 12/18] Automatically create Kibana visualizations --- monitor/monitor_start.py | 7 ++ monitor/visualizations.py | 139 ++++++++++++++++++++++++++++++++++++++ monitor/visualizer.py | 95 ++++++++++++++++++++++++++ 3 files changed, 241 insertions(+) create mode 100644 monitor/visualizations.py create mode 100644 monitor/visualizer.py diff --git a/monitor/monitor_start.py b/monitor/monitor_start.py index a1040b81..0be38e5b 100644 --- a/monitor/monitor_start.py +++ b/monitor/monitor_start.py @@ -1,5 +1,6 @@ from monitor import MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF from monitor.searcher import Searcher +from monitor.visualizer import Visualizer from common.config_loader import ConfigLoader from common.logger import Logger @@ -34,6 +35,12 @@ def main(command_line_conf): searcher = Searcher(es_host, es_port, api_host, api_port, DATA_DIR, log_file, cloud_id, auth_user, auth_pw) searcher.start() + kibana_host = mon_conf.get("KIBANA_HOST") + kibana_port = mon_conf.get("KIBANA_PORT") + + visualizer = Visualizer(kibana_host, kibana_port, auth_user, auth_pw, max_users) + visualizer.create_dashboard() + if __name__ == "__main__": # TODO: command_line_conf = {} diff --git a/monitor/visualizations.py b/monitor/visualizations.py new file mode 100644 index 00000000..e1551e93 --- /dev/null +++ b/monitor/visualizations.py @@ -0,0 +1,139 @@ +index_pattern = { + "attributes" : { + "timeFieldName" : "doc.time", + "title" : "logs*" + } +} + +visualizations = { + "available_user_slots_visual": { + "attributes" : { + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"exists\\\":{\\\"field\\\":\\\"doc.response.available_slots\\\"}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"exists\":{\"field\":\"doc.response.available_slots\"}},\"size\":1,\"sort\":[{\"doc.time\":{\"order\":\"desc\",\"unmapped_type\":\"date\"}}]}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "visState" : "{\"title\":\"Available user slots\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"dimensions\":{\"series\":[{\"accessor\":0,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"number\",\"missingBucketLabel\":\"Missing\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\"}}},\"label\":\"doc.response.available_slots: Descending\",\"params\":{}}],\"x\":null,\"y\":[{\"accessor\":1,\"aggType\":\"max\",\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Available user slots\",\"params\":{}}]},\"gauge\":{\"alignment\":\"automatic\",\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":200}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Arc\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"outline\":false,\"percentageMode\":true,\"scale\":{\"color\":\"rgba(105,112,125,0.2)\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"rgba(105,112,125,0.2)\",\"fontSize\":60,\"labelColor\":false,\"subText\":\"\"},\"type\":\"meter\",\"useRanges\":false,\"verticalSplit\":false},\"isDisplayWarning\":false,\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.response.available_slots\",\"customLabel\":\"Available user slots\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"doc.response.available_slots\",\"orderBy\":\"custom\",\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"max\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"doc.time\"}},\"order\":\"desc\",\"size\":1,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}", + "uiStateJSON" : "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 100\":\"#E0752D\"}}}", + "version" : 1, + "title" : "Available user slots", + "description" : "This is how many user slots are used up compared to the maximum number of users this watchtower can hold." + }, + "references" : [ + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", + "type" : "index-pattern" + }, + { + "type" : "index-pattern", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09" + } + ] + }, + "Total_stored_appointments_visual": { + "attributes" : { + "visState" : "{\"title\":\"Total appointments\",\"type\":\"metric\",\"params\":{\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"type\":\"range\",\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}},\"dimensions\":{\"metrics\":[{\"type\":\"vis_dimension\",\"accessor\":0,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}}}]},\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.watcher_appts\",\"customLabel\":\"Appointments in watcher\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.responder_appts\",\"customLabel\":\"Appointments in responder\"}}]}", + "description" : "", + "version" : 1, + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "uiStateJSON" : "{}", + "title" : "Total appointments" + }, + "references" : [ + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "type" : "index-pattern", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index" + } + ] + }, + "register_requests_visual": { + "attributes" : { + "description" : "", + "version" : 1, + "title" : "register requests", + "visState" : "{\"title\":\"register requests\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", + "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#F9934E\"}}}", + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received register request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received register request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references" : [ + { + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", + "type" : "index-pattern", + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09" + }, + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type" : "index-pattern" + } + ] + }, + "add_appointment_requests_visual": { + "attributes" : { + "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#CCA300\"}}}", + "description" : "", + "visState" : "{\"title\":\"add_appointment requests\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"json\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", + "title" : "add_appointment requests", + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received add_appointment request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received add_appointment request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "version" : 1 + }, + "references" : [ + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "type" : "index-pattern", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index" + }, + { + "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "type" : "index-pattern" + } + ] + }, + "get_appointment_requests_visual": { + "attributes" : { + "version" : 1, + "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#F2C96D\"}}}", + "description" : "", + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received get_appointment request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received get_appointment request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "visState" : "{\"title\":\"get_appointment requests\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"json\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", + "title" : "get_appointment requests" + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "5f36ea30-97e9-11ea-9cf8-038b68181f09" + }, + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern", + "id": "5f36ea30-97e9-11ea-9cf8-038b68181f09" + } + ] + } +} + +dashboard = { + "attributes" : { + "version" : 1, + "title" : "Teos System Monitor", + "optionsJSON" : "{\"hidePanelTitles\":false,\"useMargins\":true}", + "panelsJSON" : "[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"d76b23fc-83b8-49a8-baf4-b1d180d857ca\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"d76b23fc-83b8-49a8-baf4-b1d180d857ca\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"ce940ff4-87f4-4fe8-b283-c392c08fe0d4\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"ce940ff4-87f4-4fe8-b283-c392c08fe0d4\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"2b7f4ad4-65b4-42de-af6c-4aaf79d8f4a2\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"2b7f4ad4-65b4-42de-af6c-4aaf79d8f4a2\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"d7b38be5-9516-4e2b-a86f-5c58f1eb750e\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"d7b38be5-9516-4e2b-a86f-5c58f1eb750e\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"8ae3cfd9-faa7-4e3e-920a-6a5705b864e9\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"8ae3cfd9-faa7-4e3e-920a-6a5705b864e9\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_4\"}]", + "hits" : 0, + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + }, + "description" : "", + "timeRestore" : False + } +} diff --git a/monitor/visualizer.py b/monitor/visualizer.py new file mode 100644 index 00000000..98ab94c4 --- /dev/null +++ b/monitor/visualizer.py @@ -0,0 +1,95 @@ +import json +import requests + +from monitor.searcher import LOG_PREFIX +from common.logger import Logger + +from monitor.visualizations import index_pattern, visualizations, dashboard + +logger = Logger(actor="Visualizer", log_name_prefix=LOG_PREFIX) + + +class Visualizer: + def __init__(self, kibana_host, kibana_port, auth_user, auth_pw, max_users): + self.kibana_endpoint = "{}:{}".format(kibana_host, kibana_port) + self.saved_obj_endpoint = "{}/api/saved_objects/".format(self.kibana_endpoint) + self.auth_user = auth_user + self.auth_pw = auth_pw + self.auth = (self.auth_user, self.auth_pw) + self.headers = headers = { + "Content-Type": "application/json", + "kbn-xsrf": "true" + } + self.max_users = max_users + + def create_dashboard(self): + # Create index pattern to pull Elasticsearch data into Kibana. + if not self.exists("index-pattern", "title", index_pattern.get("attributes").get("title")): + resp = self.create_saved_object("index-pattern", index_pattern.get("attributes"), []) + + index_id = resp.get("id") + + visuals = [] + panelCount = 0 + + for key, value in visualizations.items(): + if not self.exists("visualization", "title", value.get("attributes").get("title")): + if key == "available_user_slots_visual": + visState_json = json.loads(value["attributes"]["visState"]) + visState_json["params"]["gauge"]["colorsRange"][0]["to"] = self.max_users + value["attributes"]["visState"] = json.dumps(visState_json) + + for ref in value.get("references"): + ref["id"] = index_id + + resp = self.create_saved_object("visualization", value.get("attributes"), value.get("references")) + + visual_info = { + "name": "panel_{}".format(panelCount), + "id": resp.get("id"), + "type": "visualization" + } + visuals.append(visual_info) + panelCount += 1 + + if not self.exists("dashboard", "title", dashboard.get("attributes").get("title")): + self.create_saved_object("dashboard", dashboard.get("attributes"), visuals) + + def exists(self, obj_type, search_field, search): + endpoint = "{}{}".format(self.saved_obj_endpoint, "_find") + + data = { + "type": obj_type, + "search_fields": search_field, + "search": search, + "default_search_operator": "AND" + } + + response = requests.get(endpoint, params=data, headers=self.headers, auth=self.auth) + + response_json = response.json() + + if response.status_code == 200: + if response_json.get("total") == 0: + return False + else: + return True + + def create_saved_object(self, obj_type, attributes, references): + endpoint = "{}{}".format(self.saved_obj_endpoint, obj_type) + + data = { + "attributes": attributes + } + + if len(references) > 0: + data["references"] = references + + data = json.dumps(data) + + response = requests.post(endpoint, data=data, headers=self.headers, auth=self.auth) + + # log when an item is created. + logger.info("New Kibana saved object was created", response.text) + + return response.json() From bca88530e1fdabc4bbbc958638741448318315b8 Mon Sep 17 00:00:00 2001 From: Turtle Date: Tue, 19 May 2020 23:28:21 -0400 Subject: [PATCH 13/18] Add monitor docs --- monitor/README.md | 34 ++++++++++++++++++++++++++++++++++ monitor/requirements.txt | 2 ++ monitor/sample-monitor.conf | 14 ++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 monitor/README.md create mode 100644 monitor/requirements.txt create mode 100644 monitor/sample-monitor.conf diff --git a/monitor/README.md b/monitor/README.md new file mode 100644 index 00000000..e8884e86 --- /dev/null +++ b/monitor/README.md @@ -0,0 +1,34 @@ +This is a system monitor for viewing available user slots, appointments, and other data related to Teos. Data is loaded and searched using Elasticsearch and visualized using Kibana to produce something like this: + +![Dashboard example](https://ibb.co/ypBtfdM) + +### Prerequisites + +Need to already be running a bitcoin node and a Teos watchtower. (See: https://github.com/talaia-labs/python-teos) + +### Installation + +Install and run both Elasticsearch and Kibana, which both need to be running for this visualization tool to work. + +https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html +https://www.elastic.co/guide/en/kibana/current/install.html + +### Dependencies + +Install the dependencies by running: + +```pip install -r requirements.txt``` + +### Config + +It is also required to create a config file in this directory. `sample-monitor.conf` in this directory provides an example. + +Create a file named `monitor.conf` in this directory with the correct configuration values, including the correct host and port where Elasticsearch and Kibana are running, either on localhost or on another host. + +### Run it + +Follow the same instructions as shown here for running the module: https://github.com/talaia-labs/python-teos/blob/master/INSTALL.md + +In short, run it with: + +```python3 -m monitor.monitor_start``` diff --git a/monitor/requirements.txt b/monitor/requirements.txt new file mode 100644 index 00000000..b778431f --- /dev/null +++ b/monitor/requirements.txt @@ -0,0 +1,2 @@ +elasticsearch +requests diff --git a/monitor/sample-monitor.conf b/monitor/sample-monitor.conf new file mode 100644 index 00000000..52d7fffe --- /dev/null +++ b/monitor/sample-monitor.conf @@ -0,0 +1,14 @@ +[Teos] +DATA_DIR = ~/.teos +LOG_FILE = teos.log +CONF_FILE_NAME = teos.conf + +[Elastic] +ES_HOST = localhost +ES_PORT = 9200 +KIBANA_HOST = localhost +KIBANA_PORT = 9243 + +# CLOUD_ID = System_monitor:sdiohafdlkjfl +# AUTH_USER = elastic +# AUTH_PW = password From a573cb0a37d48e156a8655e0fb1eab3c5a0bd532 Mon Sep 17 00:00:00 2001 From: Turtle Date: Tue, 19 May 2020 23:30:58 -0400 Subject: [PATCH 14/18] Change searcher to data loader --- monitor/__init__.py | 4 +- monitor/{searcher.py => data_loader.py} | 92 ++++++++++--------- monitor/monitor_start.py | 8 +- .../{test_searcher.py => test_data_loader.py} | 22 ++--- 4 files changed, 68 insertions(+), 58 deletions(-) rename monitor/{searcher.py => data_loader.py} (77%) rename monitor/test/{test_searcher.py => test_data_loader.py} (78%) diff --git a/monitor/__init__.py b/monitor/__init__.py index 6c0ec59d..401e6972 100644 --- a/monitor/__init__.py +++ b/monitor/__init__.py @@ -6,9 +6,9 @@ MONITOR_DEFAULT_CONF = { "ES_HOST": {"value": "", "type": str}, "ES_PORT": {"value": 9200, "type": int}, + "KIBANA_HOST": {"value": "localhost", "type": str}, + "KIBANA_PORT": {"value": "9243", "type": int}, "CLOUD_ID": {"value": "", "type": str}, "AUTH_USER": {"value": "user", "type": str}, "AUTH_PW": {"value": "password", "type": str}, - "KIBANA_HOST": {"value": "localhost", "type": str}, - "KIBANA_PORT": {"value": "9243", "type": int}, } diff --git a/monitor/searcher.py b/monitor/data_loader.py similarity index 77% rename from monitor/searcher.py rename to monitor/data_loader.py index e3701796..f8b38987 100644 --- a/monitor/searcher.py +++ b/monitor/data_loader.py @@ -13,30 +13,35 @@ logger = Logger(actor="Searcher", log_name_prefix=LOG_PREFIX) -class Searcher: +class DataLoader: """ - The :class:`Searcher` is in charge of the monitor's Elasticsearch functionality for loading and searching through data. + The :class:`DataLoader` is in charge of the monitor's Elasticsearch functionality for loading and searching through data. Args: - host (:obj:`str`): The host Elasticsearch is running on. - port (:obj:`int`): The port Elasticsearch is runnning on. + es_host (:obj:`str`): The host Elasticsearch is listening on. + es_port (:obj:`int`): The port Elasticsearch is listening on. + api_host (:obj:`str`): The host Teos is listening on. + api_port (:obj:`int`): The port Teos is listening on. + log_file (:obj:`int`): The path to the log file ES will pull data from. cloud_id (:obj:`str`): Elasticsearch cloud id, if Elasticsearch Cloud is being used. auth_user (:obj:`str`): Elasticsearch Cloud username, if Elasticsearch Cloud is being used. auth_pw (:obj:`str`): Elasticsearch Cloud password, if Elasticsearch Cloud is being used. Attributes: - host (:obj:`str`): The host Elasticsearch is running on. - port (:obj:`int`): The port Elasticsearch is runnning on. + es_host (:obj:`str`): The host Elasticsearch is running on. + es_port (:obj:`int`): The port Elasticsearch is runnning on. cloud_id (:obj:`str`): Elasticsearch cloud id, if Elasticsearch Cloud is being used. auth_user (:obj:`str`): Elasticsearch Cloud username, if Elasticsearch Cloud is being used. auth_pw (:obj:`str`): Elasticsearch Cloud password, if Elasticsearch Cloud is being used. es (:obj:`Elasticsearch `): The Elasticsearch client for searching for data to be visualized. index_client (:obj:`IndicesClient `): The index client where log data is stored. log_path (:obj:`str`): The path to the log file where log file will be pulled from and analyzed by ES. + api_host (:obj:`str`): The host Teos is listening on. + api_port (:obj:`int`): The port Teos is listening on. """ - def __init__(self, es_host, es_port, api_host, api_port, teos_dir, log_file, cloud_id=None, auth_user=None, auth_pw=None): + def __init__(self, es_host, es_port, api_host, api_port, log_file, cloud_id=None, auth_user=None, auth_pw=None): self.es_host = es_host self.es_port = es_port self.es_cloud_id = cloud_id @@ -57,19 +62,17 @@ def __init__(self, es_host, es_port, api_host, api_port, teos_dir, log_file, clo self.api_port = api_port def start(self): - """Starts Elasticsearch and compiles data to be visualized in Kibana""" + """Loads data to be visualized in Kibana""" - # self.delete_index("logs") + self.delete_index("logs") # Pull the watchtower logs into Elasticsearch. - # self.create_index("logs") - #log_data = self.load_logs(self.log_path) - #self.index_data_bulk("logs", log_data) + self.create_index("logs") + log_data = self.load_logs(self.log_path) + self.index_data_bulk("logs", log_data) - # Search for the data we need to visualize a graph. - # self.load_and_index_other_data() - - # self.search_logs("message", ["logs"]) + # Grab the other data we need to visualize a graph. + self.load_and_index_other_data() def create_index(self, index): """ @@ -140,11 +143,23 @@ def load_and_index_other_data(self): watcher_appts = num_appts[0] responder_appts = num_appts[1] + # self.es.search for the watcher_appts doc... if it exists, then update the item. + # index current number of appointments in watcher and responder self.index_item("logs", "watcher_appts", watcher_appts) self.index_item("logs", "responder_appts", responder_appts) def index_item(self, index, field, value): + """ + Indexes logs in elasticsearch so they can be searched. + + Args: + index (:obj:`str`): The index to which we want to load data. + field (:obj:`str`): The field of the data to be loaded. + value (:obj:`str`): The value of the data to be loaded. + + """ + body = { "doc.{}".format(field): value, "doc.time": time.time() @@ -175,7 +190,8 @@ def index_data_bulk(self, index, data): Indexes logs in elasticsearch so they can be searched. Args: - logs (:obj:`list`): A list of logs in dict form. + index (:obj:`str`): The index to which we want to load data. + data (:obj:`list`): A list of data in dict form. Returns: response (:obj:`tuple`): The first value of the tuple equals the number of the logs data was entered successfully. If there are errors the second value in the tuple includes the errors. @@ -194,6 +210,13 @@ def index_data_bulk(self, index, data): return response def get_num_appointments(self): + """ + Gets number of appointments the tower is storing in the watcher and responder, so we can load this data into Elasticsearch. + + Returns: + :obj:`list`: A list where the 0th element describes # of watcher appointments and the 1st element describes # of responder appointments. + """ + teos_url = "http://{}:{}".format(self.api_host, self.api_port) resp = teos_cli.get_all_appointments(teos_url) @@ -205,6 +228,17 @@ def get_num_appointments(self): return [watcher_appts, responder_appts] + def delete_index(self, index): + """ + Deletes the chosen index of Elasticsearch. + + Args: + index (:obj:`str`): The ES index to delete. + """ + + results = self.index_client.delete(index) + + # For testing purposes... def search_logs(self, field, keyword, index): """ Searches Elasticsearch for data with a certain field and keyword. @@ -243,27 +277,3 @@ def get_all_logs(self): return results - - def delete_index(self, index): - - results = self.index_client.delete(index) - - - def delete_all_by_index(self, index): - """ - Deletes all logs in the chosen index of Elasticsearch. - - Args: - index (:obj:`str`): The index in Elasticsearch. - - Returns: - :obj:`dict`: A dict describing how many items were deleted and including any deletion failures. - - """ - - body = { - "query": { "match_all": {} } - } - results = self.es.delete_by_query(index, body) - - return results diff --git a/monitor/monitor_start.py b/monitor/monitor_start.py index 0be38e5b..13a18db2 100644 --- a/monitor/monitor_start.py +++ b/monitor/monitor_start.py @@ -1,5 +1,5 @@ from monitor import MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF -from monitor.searcher import Searcher +from monitor.data_loader import DataLoader from monitor.visualizer import Visualizer from common.config_loader import ConfigLoader @@ -31,9 +31,9 @@ def main(command_line_conf): auth_user = mon_conf.get("AUTH_USER") auth_pw = mon_conf.get("AUTH_PW") - # Create and start searcher. - searcher = Searcher(es_host, es_port, api_host, api_port, DATA_DIR, log_file, cloud_id, auth_user, auth_pw) - searcher.start() + # Create and start data loader. + dataLoader = DataLoader(es_host, es_port, api_host, api_port, log_file, cloud_id, auth_user, auth_pw) + dataLoader.start() kibana_host = mon_conf.get("KIBANA_HOST") kibana_port = mon_conf.get("KIBANA_PORT") diff --git a/monitor/test/test_searcher.py b/monitor/test/test_data_loader.py similarity index 78% rename from monitor/test/test_searcher.py rename to monitor/test/test_data_loader.py index 853c8976..90218577 100644 --- a/monitor/test/test_searcher.py +++ b/monitor/test/test_data_loader.py @@ -3,7 +3,7 @@ import pytest from monitor import MONITOR_DIR, MONITOR_CONF, MONITOR_DEFAULT_CONF -from monitor.searcher import Searcher +from monitor.data_loader import DataLoader from common.config_loader import ConfigLoader @@ -34,48 +34,48 @@ @pytest.fixture(scope="module") -def searcher(): - searcher = Searcher(es_host, es_port, api_host, api_port, DATA_DIR, log_file, cloud_id, auth_user, auth_pw) +def dataLoader(): + dataLoader = DataLoader(es_host, es_port, api_host, api_port, log_file, cloud_id, auth_user, auth_pw) - return searcher + return dataLoader -def test_load_logs(searcher): +def test_load_logs(dataLoader): # Create a temporary file with some test logs inside. with open("test_log_file", "w") as f: for log in test_log_data: f.write(json.dumps(log) + "\n") # Make sure load_logs function returns the logs in list form. - log_data = searcher.load_logs("test_log_file") + log_data = dataLoader.load_logs("test_log_file") assert len(log_data) == 2 # Delete the temporary file. os.remove("test_log_file") -def test_load_logs_err(searcher): +def test_load_logs_err(dataLoader): # If file doesn't exist, load_logs should throw an error. with pytest.raises(FileNotFoundError): - searcher.load_logs("nonexistent_log_file") + dataLoader.load_logs("nonexistent_log_file") # TODO: Test if it raises an error if the file is empty. # NOTE/TODO: Elasticsearch needs to be running for this test to work. -def test_index_data_bulk(searcher): +def test_index_data_bulk(dataLoader): json_logs = [] for log in test_log_data: json_logs.append(log) - response = searcher.index_data_bulk("test-logs", json_logs) + response = dataLoader.index_data_bulk("test-logs", json_logs) assert type(response) is tuple assert len(response) == 2 assert response[0] == 2 # Delete test logs from elasticsearch that were indexed. - searcher.delete_index("test-logs") + dataLoader.delete_index("test-logs") # TODO: Test that a invalid data sent to index_logs is handled correctly. From 5ec78d42028e1d1d6195051a70a62a57ebf285ce Mon Sep 17 00:00:00 2001 From: Turtle Date: Tue, 19 May 2020 23:52:26 -0400 Subject: [PATCH 15/18] Add monitor conf to gitignore --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 25db94ea..c7750a2c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .vscode/ .idea/ -.venv/ +.venv* *.log .DS_Store bitcoin.conf* @@ -18,5 +18,5 @@ htmlcov docs/ .teos .teos_cli - +*.orig /monitor/monitor.conf From 49f3270aa6bd1ae2f1dd2f1b7ac50b15fca775bb Mon Sep 17 00:00:00 2001 From: Turtle Date: Mon, 8 Jun 2020 18:19:01 -0400 Subject: [PATCH 16/18] Fix bugs in creating dashboard --- monitor/__init__.py | 4 ++-- monitor/data_loader.py | 7 +++--- monitor/monitor_start.py | 3 ++- monitor/visualizer.py | 47 ++++++++++++++++++++++++++++++++-------- 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/monitor/__init__.py b/monitor/__init__.py index 401e6972..093d1dcc 100644 --- a/monitor/__init__.py +++ b/monitor/__init__.py @@ -1,13 +1,13 @@ import os -MONITOR_DIR = os.getcwd() + "/monitor/" +MONITOR_DIR = os.path.expanduser("~/.teos_monitor/") MONITOR_CONF = "monitor.conf" MONITOR_DEFAULT_CONF = { "ES_HOST": {"value": "", "type": str}, "ES_PORT": {"value": 9200, "type": int}, "KIBANA_HOST": {"value": "localhost", "type": str}, - "KIBANA_PORT": {"value": "9243", "type": int}, + "KIBANA_PORT": {"value": 9243, "type": int}, "CLOUD_ID": {"value": "", "type": str}, "AUTH_USER": {"value": "user", "type": str}, "AUTH_PW": {"value": "password", "type": str}, diff --git a/monitor/data_loader.py b/monitor/data_loader.py index f8b38987..aef7442a 100644 --- a/monitor/data_loader.py +++ b/monitor/data_loader.py @@ -1,16 +1,14 @@ import json -import os import time from elasticsearch import Elasticsearch, helpers from elasticsearch.client import IndicesClient -from elasticsearch.helpers.errors import BulkIndexError from cli import teos_cli from common.logger import Logger LOG_PREFIX = "System Monitor" -logger = Logger(actor="Searcher", log_name_prefix=LOG_PREFIX) +logger = Logger(actor="Data loader", log_name_prefix=LOG_PREFIX) class DataLoader: @@ -64,7 +62,8 @@ def __init__(self, es_host, es_port, api_host, api_port, log_file, cloud_id=None def start(self): """Loads data to be visualized in Kibana""" - self.delete_index("logs") + if self.index_client.exists("logs"): + self.delete_index("logs") # Pull the watchtower logs into Elasticsearch. self.create_index("logs") diff --git a/monitor/monitor_start.py b/monitor/monitor_start.py index 13a18db2..3357187c 100644 --- a/monitor/monitor_start.py +++ b/monitor/monitor_start.py @@ -4,6 +4,7 @@ from common.config_loader import ConfigLoader from common.logger import Logger +from common.tools import setup_data_folder from teos import DATA_DIR, DEFAULT_CONF, CONF_FILE_NAME @@ -16,6 +17,7 @@ def main(command_line_conf): # Pull in Teos's config file to retrieve some of the data we need. conf_loader = ConfigLoader(DATA_DIR, CONF_FILE_NAME, DEFAULT_CONF, command_line_conf) conf = conf_loader.build_config() + setup_data_folder(MONITOR_DIR) max_users = conf.get("DEFAULT_SLOTS") api_host = conf.get("API_BIND") @@ -42,7 +44,6 @@ def main(command_line_conf): visualizer.create_dashboard() if __name__ == "__main__": - # TODO: command_line_conf = {} main(command_line_conf) diff --git a/monitor/visualizer.py b/monitor/visualizer.py index 98ab94c4..9df790a6 100644 --- a/monitor/visualizer.py +++ b/monitor/visualizer.py @@ -1,7 +1,7 @@ import json import requests -from monitor.searcher import LOG_PREFIX +from monitor.data_loader import LOG_PREFIX from common.logger import Logger from monitor.visualizations import index_pattern, visualizations, dashboard @@ -11,7 +11,7 @@ class Visualizer: def __init__(self, kibana_host, kibana_port, auth_user, auth_pw, max_users): - self.kibana_endpoint = "{}:{}".format(kibana_host, kibana_port) + self.kibana_endpoint = "http://{}:{}".format(kibana_host, kibana_port) self.saved_obj_endpoint = "{}/api/saved_objects/".format(self.kibana_endpoint) self.auth_user = auth_user self.auth_pw = auth_pw @@ -23,17 +23,22 @@ def __init__(self, kibana_host, kibana_port, auth_user, auth_pw, max_users): self.max_users = max_users def create_dashboard(self): - # Create index pattern to pull Elasticsearch data into Kibana. - if not self.exists("index-pattern", "title", index_pattern.get("attributes").get("title")): - resp = self.create_saved_object("index-pattern", index_pattern.get("attributes"), []) + index_id = None + + # Find index pattern id if it exists. If it does not, create one to pull Elasticsearch data into Kibana. + index_pattern_json = self.find("index-pattern", "title", index_pattern.get("attributes").get("title")) - index_id = resp.get("id") + if index_pattern_json.get("total") == 0: + resp = self.create_saved_object("index-pattern", index_pattern.get("attributes"), []) + index_id = resp.get("id") + else: + index_id = index_pattern_json.get("saved_objects")[0].get("id") visuals = [] panelCount = 0 - for key, value in visualizations.items(): - if not self.exists("visualization", "title", value.get("attributes").get("title")): + for key, value in visualizations.items(): + if not self.exists("visualization", "title", value.get("attributes").get("title")): if key == "available_user_slots_visual": visState_json = json.loads(value["attributes"]["visState"]) visState_json["params"]["gauge"]["colorsRange"][0]["to"] = self.max_users @@ -53,8 +58,32 @@ def create_dashboard(self): panelCount += 1 if not self.exists("dashboard", "title", dashboard.get("attributes").get("title")): + panels_JSON = json.loads(dashboard["attributes"]["panelsJSON"]) + + for i, panel in enumerate(panels_JSON): + visual_id = visuals[i].get("id") + panel["gridData"]["i"] = visual_id + panel["panelIndex"] = visual_id + + dashboard["attributes"]["panelsJSON"] = json.dumps(panels_JSON) + self.create_saved_object("dashboard", dashboard.get("attributes"), visuals) + def find(self, obj_type, search_field, search): + endpoint = "{}{}".format(self.saved_obj_endpoint, "_find") + + data = { + "type": obj_type, + "search_fields": search_field, + "search": search, + "default_search_operator": "AND" + } + + response = requests.get(endpoint, params=data, headers=self.headers, auth=self.auth) + + return response.json() + + def exists(self, obj_type, search_field, search): endpoint = "{}{}".format(self.saved_obj_endpoint, "_find") @@ -90,6 +119,6 @@ def create_saved_object(self, obj_type, attributes, references): response = requests.post(endpoint, data=data, headers=self.headers, auth=self.auth) # log when an item is created. - logger.info("New Kibana saved object was created", response.text) + logger.info("New Kibana saved object was created") return response.json() From 840cd2a59d7cc4cd9663c0e24880e01c1731cba2 Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 11 Jun 2020 21:38:46 -0400 Subject: [PATCH 17/18] Remove cloud config options --- monitor/__init__.py | 9 ++++----- monitor/data_loader.py | 22 ++++------------------ monitor/monitor_start.py | 7 ++----- monitor/test/test_data_loader.py | 2 +- monitor/visualizer.py | 11 ++++------- 5 files changed, 15 insertions(+), 36 deletions(-) diff --git a/monitor/__init__.py b/monitor/__init__.py index 093d1dcc..1c107808 100644 --- a/monitor/__init__.py +++ b/monitor/__init__.py @@ -4,11 +4,10 @@ MONITOR_CONF = "monitor.conf" MONITOR_DEFAULT_CONF = { - "ES_HOST": {"value": "", "type": str}, + "ES_HOST": {"value": "localhost", "type": str}, "ES_PORT": {"value": 9200, "type": int}, "KIBANA_HOST": {"value": "localhost", "type": str}, - "KIBANA_PORT": {"value": 9243, "type": int}, - "CLOUD_ID": {"value": "", "type": str}, - "AUTH_USER": {"value": "user", "type": str}, - "AUTH_PW": {"value": "password", "type": str}, + "KIBANA_PORT": {"value": 5601, "type": int}, + "API_BIND": {"value": "localhost", "type": str}, + "API_PORT": {"value": 9814, "type": int}, } diff --git a/monitor/data_loader.py b/monitor/data_loader.py index aef7442a..308c273e 100644 --- a/monitor/data_loader.py +++ b/monitor/data_loader.py @@ -21,16 +21,11 @@ class DataLoader: api_host (:obj:`str`): The host Teos is listening on. api_port (:obj:`int`): The port Teos is listening on. log_file (:obj:`int`): The path to the log file ES will pull data from. - cloud_id (:obj:`str`): Elasticsearch cloud id, if Elasticsearch Cloud is being used. - auth_user (:obj:`str`): Elasticsearch Cloud username, if Elasticsearch Cloud is being used. - auth_pw (:obj:`str`): Elasticsearch Cloud password, if Elasticsearch Cloud is being used. Attributes: es_host (:obj:`str`): The host Elasticsearch is running on. es_port (:obj:`int`): The port Elasticsearch is runnning on. cloud_id (:obj:`str`): Elasticsearch cloud id, if Elasticsearch Cloud is being used. - auth_user (:obj:`str`): Elasticsearch Cloud username, if Elasticsearch Cloud is being used. - auth_pw (:obj:`str`): Elasticsearch Cloud password, if Elasticsearch Cloud is being used. es (:obj:`Elasticsearch `): The Elasticsearch client for searching for data to be visualized. index_client (:obj:`IndicesClient `): The index client where log data is stored. log_path (:obj:`str`): The path to the log file where log file will be pulled from and analyzed by ES. @@ -39,21 +34,12 @@ class DataLoader: """ - def __init__(self, es_host, es_port, api_host, api_port, log_file, cloud_id=None, auth_user=None, auth_pw=None): + def __init__(self, es_host, es_port, api_host, api_port, log_file): self.es_host = es_host self.es_port = es_port - self.es_cloud_id = cloud_id - self.es_auth_user = auth_user - self.es_auth_pw = auth_pw - if self.es_host is not "": - self.es = Elasticsearch([ - {'host': self.es_host, 'port': self.es_port} - ]) - elif cloud_id is not "": - self.es = Elasticsearch( - cloud_id=self.es_cloud_id, - http_auth=(self.es_auth_user, self.es_auth_pw), - ) + self.es = Elasticsearch([ + {'host': self.es_host, 'port': self.es_port} + ]) self.index_client = IndicesClient(self.es) self.log_path = log_file self.api_host = api_host diff --git a/monitor/monitor_start.py b/monitor/monitor_start.py index 3357187c..1a77d4b3 100644 --- a/monitor/monitor_start.py +++ b/monitor/monitor_start.py @@ -29,18 +29,15 @@ def main(command_line_conf): es_host = mon_conf.get("ES_HOST") es_port = mon_conf.get("ES_PORT") - cloud_id = mon_conf.get("CLOUD_ID") - auth_user = mon_conf.get("AUTH_USER") - auth_pw = mon_conf.get("AUTH_PW") # Create and start data loader. - dataLoader = DataLoader(es_host, es_port, api_host, api_port, log_file, cloud_id, auth_user, auth_pw) + dataLoader = DataLoader(es_host, es_port, api_host, api_port, log_file) dataLoader.start() kibana_host = mon_conf.get("KIBANA_HOST") kibana_port = mon_conf.get("KIBANA_PORT") - visualizer = Visualizer(kibana_host, kibana_port, auth_user, auth_pw, max_users) + visualizer = Visualizer(kibana_host, kibana_port, max_users) visualizer.create_dashboard() if __name__ == "__main__": diff --git a/monitor/test/test_data_loader.py b/monitor/test/test_data_loader.py index 90218577..6ceb95d7 100644 --- a/monitor/test/test_data_loader.py +++ b/monitor/test/test_data_loader.py @@ -35,7 +35,7 @@ @pytest.fixture(scope="module") def dataLoader(): - dataLoader = DataLoader(es_host, es_port, api_host, api_port, log_file, cloud_id, auth_user, auth_pw) + dataLoader = DataLoader(es_host, es_port, api_host, api_port, log_file) return dataLoader diff --git a/monitor/visualizer.py b/monitor/visualizer.py index 9df790a6..957c5b92 100644 --- a/monitor/visualizer.py +++ b/monitor/visualizer.py @@ -10,12 +10,9 @@ class Visualizer: - def __init__(self, kibana_host, kibana_port, auth_user, auth_pw, max_users): + def __init__(self, kibana_host, kibana_port, max_users): self.kibana_endpoint = "http://{}:{}".format(kibana_host, kibana_port) self.saved_obj_endpoint = "{}/api/saved_objects/".format(self.kibana_endpoint) - self.auth_user = auth_user - self.auth_pw = auth_pw - self.auth = (self.auth_user, self.auth_pw) self.headers = headers = { "Content-Type": "application/json", "kbn-xsrf": "true" @@ -79,7 +76,7 @@ def find(self, obj_type, search_field, search): "default_search_operator": "AND" } - response = requests.get(endpoint, params=data, headers=self.headers, auth=self.auth) + response = requests.get(endpoint, params=data, headers=self.headers) return response.json() @@ -94,7 +91,7 @@ def exists(self, obj_type, search_field, search): "default_search_operator": "AND" } - response = requests.get(endpoint, params=data, headers=self.headers, auth=self.auth) + response = requests.get(endpoint, params=data, headers=self.headers) response_json = response.json() @@ -116,7 +113,7 @@ def create_saved_object(self, obj_type, attributes, references): data = json.dumps(data) - response = requests.post(endpoint, data=data, headers=self.headers, auth=self.auth) + response = requests.post(endpoint, data=data, headers=self.headers) # log when an item is created. logger.info("New Kibana saved object was created") From da8c4f8e727703f76a9092abde49b1abb5dcc069 Mon Sep 17 00:00:00 2001 From: Turtle Date: Sun, 14 Jun 2020 16:00:25 -0400 Subject: [PATCH 18/18] Change loaded data to json and add Teos logo --- monitor/kibana_data.json | 140 ++++++++++++++++++++++++++++++++++++++ monitor/visualizations.py | 139 ------------------------------------- monitor/visualizer.py | 46 ++++++++++++- 3 files changed, 184 insertions(+), 141 deletions(-) create mode 100644 monitor/kibana_data.json delete mode 100644 monitor/visualizations.py diff --git a/monitor/kibana_data.json b/monitor/kibana_data.json new file mode 100644 index 00000000..d92a7f6e --- /dev/null +++ b/monitor/kibana_data.json @@ -0,0 +1,140 @@ +{ + "index_pattern": { + "attributes" : { + "timeFieldName" : "doc.time", + "title" : "logs*" + } + }, + "visualizations": { + "available_user_slots_visual": { + "attributes" : { + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"exists\\\":{\\\"field\\\":\\\"doc.response.available_slots\\\"}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"exists\":{\"field\":\"doc.response.available_slots\"}},\"size\":1,\"sort\":[{\"doc.time\":{\"order\":\"desc\",\"unmapped_type\":\"date\"}}]}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "visState" : "{\"title\":\"Available user slots\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"dimensions\":{\"series\":[{\"accessor\":0,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"number\",\"missingBucketLabel\":\"Missing\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\"}}},\"label\":\"doc.response.available_slots: Descending\",\"params\":{}}],\"x\":null,\"y\":[{\"accessor\":1,\"aggType\":\"max\",\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Available user slots\",\"params\":{}}]},\"gauge\":{\"alignment\":\"automatic\",\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":200}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Arc\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"outline\":false,\"percentageMode\":true,\"scale\":{\"color\":\"rgba(105,112,125,0.2)\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"rgba(105,112,125,0.2)\",\"fontSize\":60,\"labelColor\":false,\"subText\":\"\"},\"type\":\"meter\",\"useRanges\":false,\"verticalSplit\":false},\"isDisplayWarning\":false,\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.response.available_slots\",\"customLabel\":\"Available user slots\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"doc.response.available_slots\",\"orderBy\":\"custom\",\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"max\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"doc.time\"}},\"order\":\"desc\",\"size\":1,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}", + "uiStateJSON" : "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 100\":\"#E0752D\"}}}", + "version" : 1, + "title" : "Available user slots", + "description" : "This is how many user slots are used up compared to the maximum number of users this watchtower can hold." + }, + "references" : [ + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", + "type" : "index-pattern" + }, + { + "type" : "index-pattern", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09" + } + ] + }, + "Total_stored_appointments_visual": { + "attributes" : { + "visState" : "{\"title\":\"Total appointments\",\"type\":\"metric\",\"params\":{\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"type\":\"range\",\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}},\"dimensions\":{\"metrics\":[{\"type\":\"vis_dimension\",\"accessor\":0,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}}}]},\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.watcher_appts\",\"customLabel\":\"Appointments in watcher\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.responder_appts\",\"customLabel\":\"Appointments in responder\"}}]}", + "description" : "", + "version" : 1, + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "uiStateJSON" : "{}", + "title" : "Total appointments" + }, + "references" : [ + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "type" : "index-pattern", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index" + } + ] + }, + "register_requests_visual": { + "attributes" : { + "description" : "", + "version" : 1, + "title" : "register requests", + "visState" : "{\"title\":\"register requests\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", + "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#F9934E\"}}}", + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received register request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received register request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + } + }, + "references" : [ + { + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", + "type" : "index-pattern", + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09" + }, + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type" : "index-pattern" + } + ] + }, + "add_appointment_requests_visual": { + "attributes" : { + "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#CCA300\"}}}", + "description" : "", + "visState" : "{\"title\":\"add_appointment requests\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"json\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", + "title" : "add_appointment requests", + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received add_appointment request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received add_appointment request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "version" : 1 + }, + "references" : [ + { + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "type" : "index-pattern", + "name" : "kibanaSavedObjectMeta.searchSourceJSON.index" + }, + { + "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", + "type" : "index-pattern" + } + ] + }, + "get_appointment_requests_visual": { + "attributes" : { + "version" : 1, + "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#F2C96D\"}}}", + "description" : "", + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received get_appointment request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received get_appointment request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" + }, + "visState" : "{\"title\":\"get_appointment requests\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"json\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", + "title" : "get_appointment requests" + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "5f36ea30-97e9-11ea-9cf8-038b68181f09" + }, + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern", + "id": "5f36ea30-97e9-11ea-9cf8-038b68181f09" + } + ] + } + }, + "dashboard": { + "attributes" : { + "version" : 1, + "title" : "Teos System Monitor", + "optionsJSON" : "{\"hidePanelTitles\":false,\"useMargins\":true}", + "panelsJSON" : "[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"d76b23fc-83b8-49a8-baf4-b1d180d857ca\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"d76b23fc-83b8-49a8-baf4-b1d180d857ca\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"ce940ff4-87f4-4fe8-b283-c392c08fe0d4\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"ce940ff4-87f4-4fe8-b283-c392c08fe0d4\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"2b7f4ad4-65b4-42de-af6c-4aaf79d8f4a2\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"2b7f4ad4-65b4-42de-af6c-4aaf79d8f4a2\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"d7b38be5-9516-4e2b-a86f-5c58f1eb750e\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"d7b38be5-9516-4e2b-a86f-5c58f1eb750e\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"8ae3cfd9-faa7-4e3e-920a-6a5705b864e9\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"8ae3cfd9-faa7-4e3e-920a-6a5705b864e9\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_4\"}]", + "hits" : 0, + "kibanaSavedObjectMeta" : { + "searchSourceJSON" : "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" + }, + "description" : "", + "timeRestore" : false + } + }, + "imageUrl": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/4gKgSUNDX1BST0ZJTEUAAQEAAAKQbGNtcwQwAABtbnRyUkdCIFhZWiAH5AAEAAkACwAfABhhY3NwQVBQTAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA9tYAAQAAAADTLWxjbXMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAtkZXNjAAABCAAAADhjcHJ0AAABQAAAAE53dHB0AAABkAAAABRjaGFkAAABpAAAACxyWFlaAAAB0AAAABRiWFlaAAAB5AAAABRnWFlaAAAB+AAAABRyVFJDAAACDAAAACBnVFJDAAACLAAAACBiVFJDAAACTAAAACBjaHJtAAACbAAAACRtbHVjAAAAAAAAAAEAAAAMZW5VUwAAABwAAAAcAHMAUgBHAEIAIABiAHUAaQBsAHQALQBpAG4AAG1sdWMAAAAAAAAAAQAAAAxlblVTAAAAMgAAABwATgBvACAAYwBvAHAAeQByAGkAZwBoAHQALAAgAHUAcwBlACAAZgByAGUAZQBsAHkAAAAAWFlaIAAAAAAAAPbWAAEAAAAA0y1zZjMyAAAAAAABDEoAAAXj///zKgAAB5sAAP2H///7ov///aMAAAPYAADAlFhZWiAAAAAAAABvlAAAOO4AAAOQWFlaIAAAAAAAACSdAAAPgwAAtr5YWVogAAAAAAAAYqUAALeQAAAY3nBhcmEAAAAAAAMAAAACZmYAAPKnAAANWQAAE9AAAApbcGFyYQAAAAAAAwAAAAJmZgAA8qcAAA1ZAAAT0AAACltwYXJhAAAAAAADAAAAAmZmAADypwAADVkAABPQAAAKW2Nocm0AAAAAAAMAAAAAo9cAAFR7AABMzQAAmZoAACZmAAAPXP/bAEMABQMEBAQDBQQEBAUFBQYHDAgHBwcHDwsLCQwRDxISEQ8RERMWHBcTFBoVEREYIRgaHR0fHx8TFyIkIh4kHB4fHv/bAEMBBQUFBwYHDggIDh4UERQeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHv/CABEIAZABkAMBIgACEQEDEQH/xAAcAAEAAQUBAQAAAAAAAAAAAAAABAMFBgcIAgH/xAAaAQEBAQEBAQEAAAAAAAAAAAAAAQMCBAUG/9oADAMBAAIQAxAAAAHcoAAAAAAAB5LTeNKbroIAAAWy54aZbVwTOwAAAAAAAAAAAAAAAABjeSajrHN/6G3zQcgAAGKZXjZqvfHKfU/aqOAAAAAAAAAAAAAAAADm/oDlnRsTeWk92clDH9Dm3MV1o0uycl0l8XqyZytu3PnPLFfbRy5e6L502trdxDKAAAAAAAAAAAAAAAAa+0RsXXW12jsPXuPZXGrzm1w+R9nGpt1ieH6tvg43sD144lju08w9Xix7N9CbV93ydAZHjnr049bLbcvPAAAAAAAAAAAAAAAHz7Yjnm1HouVbH13tT8l9WN4mWv4/0aGrPNu/ZfHvecasu/V2NtPWuT/M9lix++2Hn1awH6b81vfYOjt44QIAAAAAAAAAAAAAAax2doPpgY3uxPN4tni72DG1ztD8R9a3YJtDWvp5xva0z3lrFSMVz3kWTCugf2Pxubh9Ly3fqDkjp/NeRnAAAAAAAAAAAAAAPPLnSPLelDVs7cGiegMJzhY+o9Q22y+6qofP33LaNYs+r/Y8j3P9LC2ZRdsTcc5jet66K2hy3QMYAAAAAAAAAAAAABh3PG9tE60NF06i5J6EyZoM5ExjMhgd8yAfPoNW7P5t6Y4N6zPDMi5dLDCAAAAAAAAAAAAAAYBobpXmrWhoXe0I6evXJ+wcpu9rq4cs1YXQM78aqwGsx1YbUKZJjeweW9xhAAAAAAAAAAAAAAPHMvTuCdNBDahQD78knyPJjKCAAN+6u6KyehnAAAAAAAAAAAAAAB8NU6f6Nj6OeWT4zpfh6r349UxKi1CmkRwTYhZBm21+Ee54tlOUAAAAAAAAAAAAAAAW25YEap3Bz02vQOMalGVY9G+9Pip6qikF8XqzeEz7M9HeOHROJajG1dx84dH8QOAAAAAAAAAAAAAADSO7qFcp0upuXNb4HZ9+VTx99fTy9rafiRTkpgG2eWsOoJTKVByAAAAAAAAAAAAAAHkcm9V8o6X4NX30Hr7Sqn0WvikfH35I3Tpbb3DbR5xnoAAAAAAAAAAAAAAHmwe7VV95d6S5t0oaHt4FXz8Kv3ytpevCQ9eRs7WOf8t845kersZtEQAAAAAAAAAAAAABrCVgFp1u2dEZBj/QOiRH9HlKilai+nz79kFOiDJMb+x1rzhdsC4db/bBf8oAAAAAAAAAAAAAiS9f1o2ib09+a+AVaQ9TI9G2XSpfSp8qxQJB7PANw7X5t6SwgcgAAAAAAAAAAAGHZiMLhbB0R0wOkb2dBmwrSZDk91KFdaMz1AoSJI9en4K9GbBtzHcfNvT2UxDYFdlAAAAAAAAAAAAAALVzDsfWe1FXtIhSI9s2FNhFT55mEKvQnkD78SS4kyHbOgy4x52jq6RxOsFsueEAAAAAAAAAAAAAY/fOdemMeDapkOb1YfwkmwpsK1NhTCHNhSyI+/JJsKbCt+y4c4giTYu8+SOhcpmQzAAAAAAAAAAAAWbVW7ece2LDapsOXbDPskyFMh2/ZcWSRJEerJ8p16BLi+/dtCdBrlBXoSXPdmgOiM2YDKAAAAAAAAAAACgYZoG92Teh0lUJUK1Woy48R/XlPdehWtjevKSVFmQ7UyHMIYknQZ0G1lOLOZ1t613sTzwAAAAAAAAAABpnc1trlzxLib0KmwpUW1NhVoohKlSj8X4EmwvfhUyHWKISZF+S7YYkunRuvdp4ysOAAAAAAAAAADVeyOYu0AbUAAIPdQoJfohJvwhpNMpPvwCgALn09yduTNtQZQAAAAAAAAARzUGrt9ZNo5tv/RPqNIXvaiMBueVosc2eihU9jz9+j4+jz4qiJEuwxu25sNaWjcTpoCw9Oq5Kk9Q431b5c8VyrKAAAAAf/8QAMhAAAAUDAQYGAgIBBQAAAAAAAQIDBAUABhAREhMUITNAByAiMTJQIzAVJSQWNDZBQ//aAAgBAQABBQLsjv0Syn6pJ2RiySOVVP6QR0CHci8vX9V5f8fsl3xER9JcjrhIexw1nv1XaGtv2M63Mr9J4guasMP7v9VzBrAsVhbvEzAdP6O6XPFTdgB/bYWVTRI+utiiKt4PBFO8HwCyu5mqLZyg5TxcAawlWm54mE+ikVwasTCJjeHwf2FTkw3i0ZOTdyCuNBwyeOWStv3CjIYmucRXh850V+ivpzuomvDz/dT0onGNHCy71y1iFj0lGNk6BBMtLbpMhEG7hJeKKNLIrtj2pPcUEpzjKtpzwsz9FfjjeSdWCYqdPDuZ2TZMEWpdK0pwomgm4cKP3RUwKRNPbVexDZVs8jlG6kVJBJwdFHZNGLg5YfQCOgS7jipKoTeqNmzZNujpWlPFkmyUg8UdqwpkSO/5BlTZwiqdFQqqUkQAekAWTrFiuN7E/QT7jhYjFvrC1rkIaU/dJM0H7tV4tiNYqvVG7ZNukcqjJJTU5pFXQcWC43cl9Bf7jZY4tZoD6Jt+T3RtmlEUz1wrerp3KYREao9UQQTQS0o20JZJ2mzRtZBR7IYhXHCyv0F8ON7M48Ox53pFi3dQk6ZvSKiS6YlrgVJaVTRIknpiTl0GwauJB4xZEjYTMKvxMV3wjoEqtxEjjw9Po+dIJuUJ+HWjF2rpw1M2uVYAb3DHAQ1wRmji5Eaeyzx1RCGUPa0GEenMG2IrNiLbyJ76VV3EcPvix1d3O0uimulM2qoQV0VUD5i4N+/GFhGkaFXYruoHPh8ro6768VN3A5iV+FkgHUMLt0HBV7aiFRLakUAtIaMajnxAc7LXNmKbud76/B0hvJaMgD2M/QYQKW4338hKZtodJ3vr7LrC+SJfrRzyKkm0ih5jGAoXVPgqXyWwXane+uRDiIXyt11m6jG7naYI3dHmoLnia/1PE0pdUUWnV4p1JzL+Q81iobyW74wAYs4zMxk/OAaisnsfpstlwsX393RXHtPOX8ZEvUHntmKNJPigBQ+gum3xUEQEB8hAoxtoaUDeF8sNFOJNeNZosGv0NvvuMqat9pIVIwUiyEQEMFDUTjlM4kMqQMtmjlyaItM4igii1Qtx7xrb6CTW4ePgZU8Y8SuaIOVS6IgtPZ2BcU4cQygrGT2vImfYpH+PKRnJQCNN7nhylLckOIT9zoqNvD5f830F8uNzDeUA1rZCtklaJVsp0JKKJiCYQHyWo44ed+gvtcy8idNQnlABGtPSYugGLs0JNDbI6+QOdAmukZguDln3+6T2nzNB43OGyfPsmbpqfFTpqfE46HUDQ2bCZIqEEhBAhSkL9APIFeamChqJx1Mp8laP0h6HuiX1Ez4ej/g/QgIDTk2y2H3wUdKT5nHmK3zP0i80EfkA6CbmOPDs3poBAe/EdAgFuIZTBtiK8hR0oPdbqm6KPsHIVBAT58PTf51Qy22p30+44WIswdYG6D7EDkOeE/mbmYeiiOimChmxD7M1UI52Ls76/wBzsM7DWA8VfS4JxPk6gY/8cf8AanpDFsLg3nBHQG73YuMOYd7eTriJuFlF4txNyq8ov5OZTCG8DABrQ0n6CZKIlM4uh4tHVb7ni4jvHawN2y6gqreYPUUoiUdCLVuDhRzBSRNoVT7ZvN4futUu8vh3uIrAFMNCGnkDlRg1ChERooCYyogUvk2DZth3wkz3kzBFlHTe2YlKrlKzjIcyhzYTTTMhkhtkVCaAUBETaIlyju9pcm7UARCrXdlLLOIOKWpe0WIiiUxEe9vh7v5LHxZ0kiByYSU2BMqmUMIEKcVC7B6X9SNFESmh3YPo7vpV2ViwWUMqrhzyCm3ItJkMcxyiQ1bo+7oPd31KS9TfFhv9hbvr6kd85wkG0o4HVak+TagEQF71qbCIkwvzRpoP5ThsmpssduvGOyPmPeT0gWOj1DmUUw0+Y+9ezPDzrUz6mPdnRR0M7D8mLJk+Gdd2cxSEuWUGSfZT9LbCvJvQe7zr0065vem/NPB/W0wURKNrSoSLLupZlx7a5ImOimWVvS3w65YD3edem3XV6lNOsPIabeouIsG5n7K3W7N33Q1c7/j5TBQ1M7H8tB7vOvRfk769I9Vx1qSHRRwGi1IG2VXBdlXFpv8AjovurwkuCjstQ1WOO0ekQ1WcDqvRPm769F+Tvr4dfLDn1FxbMj/HSQDqHcLqkQRmn55F/lvyTw065+Z6T6jrr4d9TCvNvgnra5syU4pp3F4SpnKhymIbJvS1w0+WEuq56+HWS82uGg/lMGybDFR0zXinyUg07dxw7Ru8XM5dZcGAckPslwUdkxhERwYwmymfZLgo6Gd6bzFiOyqIkTITuL7kNlLuIp2Zi/QUIsj2zxwRq2kHJ3jz9AEONAisNcM5rhHVcI6rhXNcOvQpqBWg/psWR20e2viU3itIs3S1IW7Lq0jZ701JWajSVpxZaTt6IJRImNJRWbUtAkkFbJa0DOgVslrdkozZuajxrA9KQUUelLYiT0rZ7IaWs1Wl7VlU6XiZJCjFMWmThRo6jXab5n2i5TmRQtZgUzeMj29AAB3CzZBYHNuxS1Q0T/Fqfr//xAAlEQACAgIBAwMFAAAAAAAAAAABAgADBBExEiFAFEFhBRMiUID/2gAIAQMBAT8B/e6mvGVCx0Jj/TlHeyKtVQ41KbkyXPbsJn0YyMBwTL8d6TpvEwAvTv3hcKNmZWW1vYcTFynrHQgmZX94KCfyl6p6fps9vErsNbbEryA4l52vSvJmPStQ+Y9qqNtMnKNx+PFB1Be4nqXjOW5/s7//xAAqEQABAgQGAgICAwEAAAAAAAABAAIDEBExBBIgMEBBBSETUSIyI0Jxkf/aAAgBAgEBPwHZIppCPrhNFSn30i6eOFDCfdBtVlasrUWUQT7cJllEc1tXOso/lifUNfPEiG9VFDsO0VPsrC4mK4HsJkRsQVbxPOx4zYgb/VQojojg1t1gMCIIzOusZhYbv5IhosJHDC4t/VYaK8xvw7k6/AZeWKw0PEtMN6xeAjYN/v8A6vFRMsX5YrvTVj/KRMU/6asKIsZ2VgWDwYwzPf7GUTgQ5PuvxeMrlF8PhYnVP8TPBYVtxVQ4cOA2jBRXMn24DLSeJVVTJg9yNuAy0yz6WUrKUGFAUkbcBpppOh564IdT0dZf9I8AItBWQTpItCyBPvwAaJrq6yaJzq8GHrfbg9KHrdZDgZfxTRQbDW03xtvHvfbUnTaZT/tVrvsEjdFdISKNkRXfaKyEiukLyKEnjveZLqRXSCK6Qk/eaKCXSF5dSKEjIim4PUyhLqRQl1J53GDdeNzMAvkWcqp11KzFZys6Or//xABAEAABAgIFBwgJBAICAwAAAAABAgMAEQQQEiFRICIxQWFxgRMjMkBQUnKxBRQwM0JikZKhNGOCwRVTJHNDg9H/2gAIAQEABj8C6k36P/8AItBVu9mulOAlKMIS4gzSoTB7FmY5edxty3S9m9vT5wGic5k2eHYr7k84iyOMJODaj7Olbh5iCwTmvJlx7Fo9EB+cwrYyfMezpfghp8fAoGErGhQn2I+oHNQbA4Q6f2T5iu26sISNZgpYSp87LhHN0ZlI2zMZ9HYUNkxATSWlsnHSItsOpcTsNdM/6jUySc5vMPDsN6kH4EEwVHSYpB/a/urOz3T0URbpDhlqSNAyeUo7qkGAy/JqkfhVVLH7KvKqkUQnSLaewwyDe6qXCqlH5B5xyhvcVchOMF1wlxxZibp5MfmOhaO2M1CRwgqcCbIgLLIE9ETaXLYYmoFOBECh0tXPjoK70Ur/AKVeVVHcncVWTx7DQwNDaaqc6sySlKZ/mFuJuaFyZ6hGYmatajWXHDICEoFyJ3JgJGqEpxMoIbRJUvrHK0aYKTOWsQ/b9+hpSVjhUFDVDD4+JA7BnFIf7yzKqk0Jm4vqRaOCROcBpsSArLjpkPOLSrk6kxyj6wkJF04/UJibKwqzfAWnQYcljBpLfQWkodTsOustE3tKl2DSXtYRIbzWukK9zbS2vZOcvKJi8Vco6dwxi24btQwruuQOkqA22JCEqacmlUFStJhFFR7x42dwrco5NzqLt47BZowPvF2juFfpOjayEFO++PUaWZSMkk6tlWe2lW8R+na+0Q2w22hJN5kIn0WhpVAbaTJIqCSTIRaV0j0Uw96QevDST9a6M/qSsT3dg8nqaTKumjwf3HrrSead6WwwGKXNbWpWtMBbSwpJwqcfXMUZBsg4ygNtpCUjQKyhqTrv4EAXuOrMhC2R0g2Ss4mWRR3sUCfX5w+93lmukoxbn+YUy8m0hQkRGtbCuguLTDqkRZpDIXtF0BNhxsDVZjpOH+EcxR1K8V0SK7CcEwEISVKOgCPWKQJ0lQ+yKUr9lXlkKa/1r6/SHe62chKf9iFJ/v8AqotPIC0K0gwXfR5tp/1nTFh5tSFYEZAKGyhvvqi0Byj2tZqpHzAJyKQz3kz6+980k5FHf7qxOARoNdl9lDg+ZM4nyBb8CjF4eO9cTaojc8VX+eQzRQb1KtHIbHeBHXwMXBkpQo86zmq9iSTICHHQebGajdkUTx9fn3XBkppDX8k4iOUYXf8AEnWMuajICFUKhKzPjcx2ZNF8U+v0hsabMxwyg4w4ptY1pMBNJaQ9tFxjnG3kcJx75f2x75X2xcpxX8Y/41FJ2rMSedk33E3DKLuptHXyk6DDrBF05p3ewkIEjP2PKrElvZ3DsDlmRz7Wj5hh7C18R0QWzr0ewFscw3es/wBQALgOwVU2gpztK2xriRuOTaV0RE6uUGn4sqy2JNjpL1CE0dhMkj89hUu+9D5+kFxPMv8AeGvfBtMlxHfReIkRKuQ0CucW0dE/iuywytw7BAc9Irsj/WnTFhpCW20jQIeVOdl5Q7Bff7jZMcrK2hfTTEzSCg4KQYudcXuRGf6OU4cbIEc16PeR/wC2CGQQjacnEaxAW5R3ncZOSi/0Wue1VqLKW3Gh4In61Legwqj0C0Su4uESik0cnSAsdg8kNLywnhpy71COn+I6Z+ke8/EXKBi6JgSyKOSblmwePYLdGbBUGk3yxMZ6FJ3jJ3RagHGBfpgCYvgjDJuhLnJrTIzBlDVIHxpB7AKuTTaOuUKZeQkhQw0QpOBlkeKEjjCN0N7oQdkJWNcXaDoyHqW4gKUlVlM9USKQeEWUJCRgOwlHbXKqWEJ8MN8YTsMeExZ1i8ZFJT+5/XYbqsEk5BgROpHGFbCDFnvROJ10xHhPnVd18k6oU933V+cUpX7SvLJO6pW+EbzCx8sTgkZFJRi3P81UxnW0+fz1+kO67MhDW8+cUo4plkSqTvgmE7zF9c9QrKe80RVT2J3OHy6+zRQb3FWjuELa1tuQGtbi8n5h+ax4q7osDjXRlnQVWTxiceuTzS/PhPr7iQc1kWB/cFxoBaVCSkHXAW4AhKeikasq0npaxkSEcodPw5AI0iDReTQlRTZU4NMqqO9O+zJW8ddceVoQkmFuq0qMzly1iJiLs1fnGfmjGLKNHnF/RGmNmrLfoZOg209d5AHOeMuFdyTF+TbHGq8xIRyaeOTorZWTmqNlXHrqXXqStKEiQQkReyp0/OqF+r0ZltxzMTJMXqNSlX2hkecWk3pMSEWR0zpOGRzk5RZ1RcZQ21SQlbTuYbQ+kZ1CbHhzfKLTD7zR+sIQpVpQEp49eFGScxgfmvxGom3fhXinWI5lMiddciuzBThU2vhUFC4iGaQNKhnb+vu0hXwi7fCnFmalGZrbRgKnDsqkmLKquUldXPEVLThfWugrNy85G/r6aA2cxq9firSNsKqcNV0cKnE/LW0rZKqWN0EVIebMlIMxDdJb0KF+w9dW98ehA2wpazNSjMmsq7or3qr4VEYpr8KqgYtd4TrNDdVzTvR2K64VKMkgTJglJ5lFyB/eQtWN1bQrNQg1Op2TrSe6ZVgi4iLKzz7dytu3rfq5eW0g9Kzrgcnyi33Dm2jkNpxvrbTgmoQakwrfVLG6JVLbxFbSKUJtKMjfCaVRKS8gjiCOuLUk80jNRWBEsBKvdUINSd8K31JO2FVJMKFaQo861mq63yLZ5564bBryBsvgnE1J3wrfUN8KqEGtKsRWhzEVpUo805muf/YmOsqdcMkpEzC6Qro6EjAZDi9kqxBO2pO+FVg7K21cK1J7t+R6o6rnWhdtHWf8fRLSm0nnCPiOEWVpKTgcgDvGtSsE1p3wqtB+WtQwNdk6FXQRhW3S2UrFk3GVxhL7X8k909YcpBbQkIEzdDj69K1TyEpGhIrUO9WDhEzWJ6q1DvCsGLQ+ITrcoDsjZzkTjMQlM8B1hHo9s5y85zd1lqkp+E37RCHWzNKxMHq7lIcMkoE4cpLmlZn7G5J+ke6c+2PcO/bH6Z37DH6Z37I/Tu/YY9y59sXtq+kaPYn0e4b0Xt7sOr/49lWai9zfhVzVHdXuTH6Wx4zKOdpDKN0zHO01w+FMozuWXvVH6QHeZxm0Jn7YzaO0P4xc2j6R0R9I0ZGgR0E/SL2Wz/GM6iMn+MX0JvhFzK07lxzdIfR9DHM01J8SJRmpad8K45yhujhOM5JG+EUhoyUgzhuktaFD6HqqktLsLIuVhFt9Tj6zeSTHNURocIuEusc6yhe9Mfp+TPyGULDNIUphfwK1H2n/xAAqEAEAAQIEBQQCAwEAAAAAAAABEQAhEDFBYSBRcYGhQJGx8FDBMNHh8f/aAAgBAQABPyH0V5DaOQNPn2/jj1BJmZQ/dDfO54fwoIkAS0ruIOkw/jn9TJV5b98PwseQ7vsr6cBH7/j6WWqzF7AufhbOZSPg/dTJ/GSIdfkpVYfK0+MgH4S652HZ8zU3LYwlztLBR4/r8mu7PU/VdCM/sUwFaH/aiTJrNjFPrGF5iT43iPwamZR1i3mnJlJWpuUh4YTBw7bd3eRTJKei7WELpWw4GDPk2epW0Pk+3vtgPpN2GQcRbln9e34OxIddF/6wHTHRnB1zc+lde3TsUOG5OdCvdVAjogoBRmSUKfvgQxQypUCEgz/aoCwC5p5daMPn8vCRIdo2fg5Yavq4PiJk6FJmmyZP76g/eU01akT86p1kj+5oELCKNO0fuaMKFk3OqphbyQ5VKAOYJlfB80KSkYmY9dfwIIsiklJOwFjBLSM30+RUSte+9NWsLUyNVyKXWPKyKue85jTSmHqLKYSQo5FrqshiLJCPZjmyfsbn4GFWGeyHziaS5GNBLyohoISJTRttNd042jyGMZL0wbdaiRnu1rVwJrSapSVobjIA1M3HVIH0cp/A59HYD+3xjAbtgJlJ3xPc1NERbMo2mkndX0YxUmirrbFCMyoU1SBZAtinyj1havCLFymMHYxuzHcqz4aLk/gJosn7835x6lFXOGMv3vQUQWzv7CsuNisF6TEjoKJScBTRDWnw5AfcaVNgZUKb3ejgv1KbqLPx68EWQTSLMyvOOwP7P9UF2eFP2Esjw71OQag2e1Rt+Ssqi/yC4HtQpG0FRxEuapOTnbpqBwCVaIGyLk5dalfT5XBOTK52b+v3lHilKXXGQG2CBeIgVq1xzLZ0daSk3JcDMXmMHbnRu2bPLpywJc3J7p+uCXWwk6Pr7BYV7jwX1s7o1onUhI47cGCkMO2XipJs/wDSUAjuRToAEBBiGRCtjLgulbxvr3+kp4ZAoQapo/wiGBKulXRl7DXgVhs8Pr1I6l+OF0Z0TLlUGNZdbvG1ICVWAqfurF4NuFzmnxPr3Kye6u4sgrERoOafbyoMkdf8KWZHV1/1dAXvkVAIvpBPFTz/ALbz78Tvl8923rxBkISohF95rL+BiCVopYiQu+v8KjMebTR+AcZpyDPVoiKJCcf2Wb0+vnVSIw5nGwKMc1/qhLAQBofgZyK/aCmQIWR04TZ3DfamVf8AKFGTMoNHZP3xHXsvn0a1YBdVzfwKwTQRXgnw+KiRtduhrSwZ+ppSEhcnBYju0LG3YwBfmcygGZ9hiTfKxeZs93V0oJlhEFZinYJt+BF5iC6xamstlNnub0T1mmD2Kne4/wC4qcm2pHzSWWeVvxUyO3H34XTaXM1pnM6J2ikhWNS+VGuQwY8Nezl/TqL/ACyAba1FlTHSz8n4G63iC74OJclGk6NX20KtNl76fptqU2NNXDU4LMnxMeY/AyFJgT9CKjbtzDhAUMktT0omKSeEFRnBDJFNooh0rJt83CFQFXQqYiJDMqyPemuv4AhuqQS051AOZcyt/bge+vBVnMTTJdXmvKfNX9PptBBoZOa4BTuRpySvkqNO5NDADkEH4FyPKluzfOKGNayvIsV7RFZhyFfY3q9OaV73gavuCEvLF9/8fgpCEYYa+04UpTvjD5kgoQN6cvM1mnIDxg1h+hFXLlRSGMxpCggdMe7MIBKEyt68raBLSKsyDpOtkfmcJS62FCQYB9BtV3Py9qSAzKsIizwQ8r+z/WE/Nj7XevvnAfUbV9H5quR/aTgPW0psw1qsoU90a+62oglAiPtjJK3MfsFQ/rAF7knX17OIAfTVqYt5bZvUxbYjYvwxdN7ZitrWXwYBNBMLqgGlfqxQGPaCKBE2Co03BO+HxSATJ9de0B1s/J8VptQL/qjX3C5OARN8q0DCUbnA7+5izgpByOfOhNnLD903ZcVDhJGr8BiZaraOGbz44fj1udc3sUmMvbdePwluUJeEqDKNc0oXkDzTQSGNbrTO0XFVwCDYcjjziiLZs/r1sBC9iXcc2HamUDgShMyi03RycM0Hq0YOVqUdjPzeEUkabUiZiYWoO2vWoPT192Wr/wA8K+CCmxDHGb5vtNZ9sGkwb8DTxI2HMqE/WqFDK0kvP+U4Ao7CKnRfMU/KdDVhRsaS/wBql5h1mqQJdklCVGCtRjq9deqgPXniXufjwiaATiZhJMylYmYWlKrLgrFyb0iGrDfMSwdFRImjT1FiORz9e0fJc9BUnw03ce/V74dL4YFS08CE4OVAeVC3l3DdtrGzvnHuPXjKeQ68nbHfWk80GMOtEMCCka+FhMNiTGPdJYQz5GVKroxhBeVulPNo3uHrUykzfUYKwjVcRDdVpSnng2MAzrP6MO5BiXsBCdGatTkGKzstz9L+sEe0jIKXF31wNzOGPWkuGRXiGD7g/FCC3w6hi7Y7hmYqWokTSgh4weTT1a788u55dKntAVkmrBwbju42uQYedXgGDjr0ILdgoHJaGXkcLy5x1xaHZeCTrWlINTkDt6tAS2Crvvsxr3xgnViiY8iwGSc2nccgYeeY9Qu2jB4NrBUW3w5WTeuRkyYFqGWf8N9XMRRYs9V+uC+8rldczCI7KmODxWKcJvQjdwLNX8puNjchxZYyrbTsoAQRuJ6krryPKmMbvCOD2cYmduWluScPAV5eBnXfDj0ROLczhwGHPa/ePUxasJz/AECnwDmEPBuaTG3nBx8XXn43HOGPT24maAlTq6oxa+SSINSrF02TPUH1DYVWCVnfbGZ0guYHfGPi4icVBzU0rzllxNOjBjZ+bGKE5jNCHIDFOIqITbWpvmC1PqD5YlpoO/qdnH+4UKU02o+nmpKt6fye3Gh/BDWYPpTJm7qGy97hp/3VJ51iOZ99ZMerpDNH8OufTa6vToCY0o5/0UC5E03HXNqEZDmNCh2+T+qL4F+U1+hr8VknV9MnPZXi0NZIuhoPKlsPaoORUHIrYKUz9qlc07K8hgr4DNZE+iKGuO9WdpvB8US+zt8FqSjPtnFZZ3Mi9yl4T2RVkQrfanom+PcPSlogIJ6qaKiw5aMLDqzfNCwA2PUQTo01OtxrQzU8T7g/yf/aAAwDAQACAAMAAAAQ88888888808888+W88888888888888888rf8888u988888888888888888hf/lQ/uW88888888888888888izxCnTsD8888888888888888qDL1izsdD/wDPPPPPPPPPPPPPPOQQXKksCw9vPPPPPPPPPPPPPPLgbxWgn9Q1vPPPPPPPPPPPPPPKgRvLrvOwU/PPPPPPPPPPPPPPAgU0+uu4QU/PPPPPPPPPPPPPPPywQQ7wQQSvPPPPPPPPPPPPPPPCSwW9yRwPPPPPPPPPPPPPPPPOkgBzx54kFfPPPPPPPPPPPPPPPCgVY1TwUw/PPPPPPPPPPPPPPPKgdy1XSQvPPPPPPPPPPPPPPPDKAUazUSVLPPPPPPPPPPPPPPPFAASy5Y4V4nvPPPPPPPPPPPPPAxQVUWUQRSW/PPPPPPPPPPPPLPwSA87wS0ALvPPPPPPPPPPPPPEAQKWUAaDU9PPPPPPPPPPPPPNY1AaLBCQdQffPPPPPPPPPPPPKQWQaIU63ACdvPPPPPPPPPPPOIAQQi3BaBQSA9vPPPPPPPPPPBCwQEgdwS0weQX/PPPPPPPPPPIQQQRzxyxwwQQcPPPPPPPPPPPET7TPrPLHPnrbRNPPPPPP/xAAmEQEAAQIGAgEFAQAAAAAAAAABABARICEwMUBBYXFRUIGxwdHw/9oACAEDAQE/EPoJHE8IjvAgmLIlHhEDndYEJd+DaXiAHgmXtyszd+YMr9gzPufyWGenp9cQKPt4ihLBENv8/cGBVfP9jBRmtvZ+TuZgbDPyHVHgFeP6dCtkMsZrdlzVrf7KWpsNj9tHgFHeKrk7y/uPW2j13dq8Ao4SjwCtpalqvANFeCONeFaWw2lo8Ag3xsW/BMbtwjG7cG2UNAOU65RxMeAUalGjrhQozqG+FNYwODqFHWKlOqNXWKsNF1DVdS9peXl9C8vL4//EACURAQACAQMFAAEFAAAAAAAAAAEAETEQIUEgMEBRYXGBkaHB0f/aAAgBAgEBPxDs7Q6RaEFq8KgOqwSpvwtxdF4AzFMRtxMiC/CCosTQyxlOj25/aAJZMvEN7ZwRodj3n9JZr8nJEsqO3gBbUCiozCqWfXm/p6hAKsEL7r/j4RNaA+VBgUVeLPTxLWcsfPsIaXgC9BsbOHkfZKRauBh/xiogK8u67BXLPQDB/b9lmyvrH6/JuZeR/o+aDD4Ay6bJbYiPvEtKVvZD9sfxFrT8n/KlM49BLdzQX4S5tBGGLZdLLab14D0JezFI+E+ERmAKNHS8C90Bc9jo43hYEcaPqDwxKl1CNote/gG0I5cAgViVocUxUAbhrwFy9SSpRE0qXPR8Hlp8hzOZfOmXwQymDrxOdc8Fj4G6k2B0IlTMC2LcSyo1riU13hbWlaEGtmWYI7bdFHeCjZMgxVzHiIm8EwzbDMC2phgp2lx3RWXfpLgXHcYEo3QC06ZzcGVKiV3rWmd6ZQymLTAZuE043estcaGydMmGUyJlodMSmpYLPAR2BBYi26smYOnvEspiKnuMV1DTKKm9Lw0VtxU6Z/DSxpO5a31WSyX03l9wLRFcEV5i3Mt6bYFzA+Yc0C5JS9ur/8QAKhABAAECBAUEAgMBAAAAAAAAAREAITFBUWEQcYGRoSBAscFQ8NHh8TD/2gAIAQEAAT8Q9lKef6MhW6C/t/ycq6cFuE8qMgEMBSPZ/Cl8WRwAxalmvqwNOwf8s6KUwGeVSyGUVvP+VOn4Uswqb/1JpULs1zH/ADIZE+BSnkQgLb5DE/CwZcc5Zv6wrel+H7/526mPAn1SQJMNASdpo3ypMESfwlujYW2K7yda2Fnf+OsuBnskUOrVpv8Aam5rvQaXuRDuIfhQtm1wbvL4pCeQxvqAO1EHKYkOZlwzrmjdpfXCz5Lq8qJda6/gxEFx5mx1gUqht2KrK1tn938VOFRWrfvAZ9qveAch0Ps34GATyKQxLpwe2Ap0PASrcwRYbjcNzpWdb+cKtwLhzLe6P4OSCiA43Xng/Wgr+Kj95U31WzOph3GBWwZBkUiYXzhywKFyyzp8YVGDmQvqje5KPtU9GeQRYrKBe6d6F15a0mCDOomIFwFm2+a2xPAxGs5tFKe49PwbyaeA2nl8RwVe91gWvisbutRm3NY9aHkYgE8rQ2K2626MwGeK0DNo7h5nJeox2oxgcxoFKEhJ0gPurRWBMAwDm61hXPFFJlbJWS4ImCI6PzPBNIO2iMlXyOXJHkP4FAIBV2KYl9k53wBwFi4MJ96o7zFBzgpi7zTmtbdbdD3wVyAZtIipZlnfV3ochayu2I8tI/t/inyvEsCMl4olwLGjmO40QKdg1SWjNbaRAI6pF2njcUgi3+IunT8DFmi50PMHpSqy48DHOKA5J0EHnTFQkJEcEoXKjkiyJ5QPlypbgFFcgb78ZzjJToNaHVHdzdVzasRrDIImSkMPKzWmIx0CiF6TxsLJJcVn5dn4Fo2YI42R2HicQUByae5HWgKyyURMKOBOFSElxzKOKMJENpKnyq/tlT+SYylgkKFXWg/sPxRtWgGO7qtbdYqBseQUVVkJ5rY3o5UtmAg2R8cUcjZb3zlIAMiSfgEiMcmwvuh04mTcE6I+6fsMC23dDFzmlbhMCtH/AEKDLkjp10oGUrS0SW05KyzvRMlAwBSGVEFiBitRryhOXUPgo8EhmEtgMgqFkGapXIwOXEURMSiu9DMyfzL37OwhOxT2oc7SDwcSXbhnKKrskJk5miYjQIcYZAeMeaMzjPcjA0ORy581i5SlCgAG0vqkCHcPMVk4KcdiXzRvLdU3cWi3baJYAVYi8Vxcm7N6UDSLR3SPn0Wx0CcBh5n34rMQTvIPmkZSpV4hDnUsAo4VdGsEj+d6lsyqJt4Q2b0ol4VvzjxLsFMRQlo9Q36KlT0Bp1ByeaGrDm8KZ+XoWBNRjA+H36jgK3Bnx6FkBdLGcDstHiIBgiSPF625I8lOE26i9yh2o8B2B4GjZsYDdRTHSjRgwAgON29hOXHkvb0NZiveZfXvwFgd6C+vQWZKOyIxsTtxbpx6+p8xHoAXVpUrYvA7mX0MljN1D34NSbKGft6Y8Yvm+X05NJREQhpk03weEVHoR7wcAxVcKXRNdgOfynP0gVM65De/t+ZgxSHxTZj0s8GWC7Ym1WszLSc4uehUxDzEHdU2djTP3v4pS6kZPeoW/sRbxJe5UoqZB7wX6j6pwsqjC2+/fkQRdmJDSrBUSzpR8dP+CVFQBSYEYMA2H/F84aBcbHrd6n4CVHoWo5hids6ZsiESEfWrgsINGdAUXU7kf5pmSJCeuAZd7RuF1zaE0Pch0AEAfgRPSgMWZ66lIfFAhRiJ6b+C7DFZCouybAYDIKJsiSJlUBcAPJ+j6in0wp0HPZQmgS/Ntmv4EEWAS0EgK0zjjooOYpY8/wCqSGmbBYFjVC/UUzEcQhOBNgMUwDWr6i0NXNeJOMNnwzBoTOjDNauL8SgkTq4FHkMXHZOw2JdykegQQAXXV3om4nszfLxT8A5UO66WYUOrBUOxRIJmeQpKaJkZpcHo0QNoHNQGtg7qSrLQqEHRVTXpgT7QC3T0hSLHAD+azoDnPODKnOazP8Q+FHmuRDvVKU8YC1k0KybxC3LqhFZ8iDiqT8CSkGEZ4p3Dr6RTCsKAaqBXwEloOJ1azf6edOWucUsSFyXs1KhwkSyaJnQKTYGHT0XKPuWg/gSl0gHmnDZSQZSvvc9Lik5AVnu73mJpQsSRlGTTXNm5kTuUmUQpMDVsNnQZRj6RrUgCVosR8iKkZjalVFoGYWdGTp+ADJBN+t2JptJVEgsmSN6dRlUdYU9DbJxzthd34rdCT4PirN17l/FYnZ86tbO5uKVihK7pZKxsjNs5dMOnoFYlGLQBzYJ2q134hJ5qXIcAEs2D8CK+AWv9cpccU5RyoE0x7FimBGAdpVv+k3+6ubYU/eGg0Xc/0HWrmzN3U+/ReD69PwTQGAzDpROMXbk2t5k8SQDeGk40KWFzyL0rWY1b/U4Vc+58lb3E6yvkpTWCnnieSm5icVObkmjiDLuFOYPo4X1UyU3GE7+/egURyAlq/mKnQHgonmGK7oHz6ERhotFkua2+JpAMVCkMOAg6Wq/aCnNWOaH6acGEEdyguVgaT6DmsDOUcIz+IJwiPmffgJzq7b5qQzKC86MMiB6L4n0EJoZ+u1BQESyNASQXC5E0ITIr5q7aLxRsBWOiuOnhK+DrSyzwN5CA3hpNRki2bMX1Pv7LQSci3g7VHuyuYED3ntRuIOljOvMekLWCs83P5psw8CZxCztwEUHWkIlLAa1MAZymenph34gFYRbAn3ouQhVbAUMby4sq+SlDkBHb3zWifNoeSYpMuliGXGTAODu1I/TwDiq4rr6JiOMzpVsouRKHgOQ/Rypsw1lFQ3l+KIkkY6n8UInEZrn0UlElWV4pFMNiIyNNnA6JRDAhZZcWKlmc6BeWWm+dc2XX3qfhMTjIh1bUll+M0L8+smm5+NzKSC6RK3JVs22jSTkwXiMaeszHybbUbsV25/NNZFj5GHrO/wAycgHuLq+9KFXYN/8AI430LUdWcHSfQpaJI1ECEwX6Q0WZKDCHwkYpdi4CiRFJH9oPTun4T8U/DmiRw1LIrYV+TDQiSXH3l63uCVlJsbZZUGB39+PBQ+sHzimibG7VKsIGgwdjhcr1dALgb+iyY0nDOKJE4JzejSNXQBRIiwGTRv6JohELXvQJWAVxRqQO1gqYKwxYy5oDrQ6x+LyBRu4QjGFkHzRM8SIBFl4n3uVFIIGNmNfQg6cbhhD6P84T72hJQNaceBwbXBJ/NBjLU7CkCKrKvBWxaKXVKeLBOvDUd33OA3zuQgyJTJRRbA9yeSe/GcTTONYOsU45BHFJeBdilCaZv/jhzCnfgLNDN2AK5vBcEpBbMzeNY4ITihq1MPBx9cNXoH88RueZWBLHMv09/IDMysxboPd24pAYE1FjgPItw5SngTVMiMUpS58BTMyJweNgXuXL9eAEt91KI678CFTy1UxycKDQiRbhZ9xn3qmikt3LPIxeVJkIyVEq8QwJ1HKmZxS8LJrPbhcDena0Dhd+pimzHCz4ovIT++GOAQoTAcO+DxCBLU2wjkC3MPeMm0WBEquQFEGsutCbo1fj0aUD93H9ib8LuYUp2vi4RHofKthEeeGvYA3/AEcbDu7kOH1xCqRSFGCUOQ82+AOo57+7RghQELBrhLHWoDCEYO8Acr6+gaSPUw+ePUq68PGfNOf3LcJLrHuJW13zcDQwU6n9U72KHbgxN8K/fFIijicDReVsKAmijS6umZLG0Sj6fdo3ASrkU8xVjbKX60vKOKYgIdWmODg6f3wgvIKkHh4T++H7LWvi/HDZ/wCSutjhd+JD3qycDM68JBYs8jZqBfoG/BKEUS4lWcVNbsHnPI+7xi+TRbMlnN09AOzHQKZfM7nhzruzNXiwR2tw/Wa15Z8HDY4vmpnsfHBQJkzRgftI4DCJiUT7cZxOuikWE9xX5TR9XEJEcE9y497oAE0RApNsOD7efo0akzd4y1wkdq/1iHheP7SV4z4OFgd6v1l8cbExa8dYF5Wf36JGABXsE5uDt7hgu02kAeA4CMczV5VilIAOY+jRy3yON/YcauD9ZKUr+7HH9Eo46sSeTbjnMYfFYwy9nGFCStgkSyJI05THKWv+JzPboJDcacwvQYJ0xacBUQuAtjoQcRIDOiIBtMHNxSZw3sDPiQYgE4SUjszjfi8BEMacZHYAbM48XRgQ6U1MbA4jhEjSdiTow9aPJqmJNvHuMNOSMFv8wvINfbKuKsEX4rWsELi26xPWKcIWFgSfPtwxSTchYN1tS31YLbCDYIOn/ATAXpXmfGvCY/1WOvl/HQmC/ppX7r9VgQ/TSvP5/DXhDH1TsNbkf8RkuZq6MhybnN9uUdlY2X8ju7UhCLQJompWD6IqHE9D9FnxRgNxhHgeaVHWTz6pB3Jjju0qNkDP5pqKnTOZ80MENh/Rrx+r6rBFyFB4H0V/mV/mUpj2qxB8zXn+b6oCObP6olJPGf46DZtzZ+EpxIs7HoqUOuUCK6QfNCqrLyD4FSQJggL0NJtH4v8AKFOTiyKvNNLCuWBitktQsQGe4Yu4+1boJ1+2szSlu7hRmVQxloQawF70qKnmAAHomp9lFJjbjId0rm8Fu5YUxJkZUMDbWdbf9P/Z" +} diff --git a/monitor/visualizations.py b/monitor/visualizations.py deleted file mode 100644 index e1551e93..00000000 --- a/monitor/visualizations.py +++ /dev/null @@ -1,139 +0,0 @@ -index_pattern = { - "attributes" : { - "timeFieldName" : "doc.time", - "title" : "logs*" - } -} - -visualizations = { - "available_user_slots_visual": { - "attributes" : { - "kibanaSavedObjectMeta" : { - "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"exists\\\":{\\\"field\\\":\\\"doc.response.available_slots\\\"}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"exists\":{\"field\":\"doc.response.available_slots\"}},\"size\":1,\"sort\":[{\"doc.time\":{\"order\":\"desc\",\"unmapped_type\":\"date\"}}]}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - }, - "visState" : "{\"title\":\"Available user slots\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"dimensions\":{\"series\":[{\"accessor\":0,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"number\",\"missingBucketLabel\":\"Missing\",\"otherBucketLabel\":\"Other\",\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\"}}},\"label\":\"doc.response.available_slots: Descending\",\"params\":{}}],\"x\":null,\"y\":[{\"accessor\":1,\"aggType\":\"max\",\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"basePath\":\"\",\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\"}}},\"label\":\"Available user slots\",\"params\":{}}]},\"gauge\":{\"alignment\":\"automatic\",\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":200}],\"gaugeColorMode\":\"None\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Arc\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":true},\"orientation\":\"vertical\",\"outline\":false,\"percentageMode\":true,\"scale\":{\"color\":\"rgba(105,112,125,0.2)\",\"labels\":false,\"show\":false,\"width\":2},\"style\":{\"bgColor\":false,\"bgFill\":\"rgba(105,112,125,0.2)\",\"fontSize\":60,\"labelColor\":false,\"subText\":\"\"},\"type\":\"meter\",\"useRanges\":false,\"verticalSplit\":false},\"isDisplayWarning\":false,\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.response.available_slots\",\"customLabel\":\"Available user slots\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"doc.response.available_slots\",\"orderBy\":\"custom\",\"orderAgg\":{\"id\":\"2-orderAgg\",\"enabled\":true,\"type\":\"max\",\"schema\":\"orderAgg\",\"params\":{\"field\":\"doc.time\"}},\"order\":\"desc\",\"size\":1,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}", - "uiStateJSON" : "{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 100\":\"#E0752D\"}}}", - "version" : 1, - "title" : "Available user slots", - "description" : "This is how many user slots are used up compared to the maximum number of users this watchtower can hold." - }, - "references" : [ - { - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", - "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", - "type" : "index-pattern" - }, - { - "type" : "index-pattern", - "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09" - } - ] - }, - "Total_stored_appointments_visual": { - "attributes" : { - "visState" : "{\"title\":\"Total appointments\",\"type\":\"metric\",\"params\":{\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"type\":\"range\",\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}},\"dimensions\":{\"metrics\":[{\"type\":\"vis_dimension\",\"accessor\":0,\"format\":{\"id\":\"number\",\"params\":{\"parsedUrl\":{\"origin\":\"https://469de2aae64a4695a2b197c687a00716.us-central1.gcp.cloud.es.io:9243\",\"pathname\":\"/app/kibana\",\"basePath\":\"\"}}}}]},\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.watcher_appts\",\"customLabel\":\"Appointments in watcher\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"doc.responder_appts\",\"customLabel\":\"Appointments in responder\"}}]}", - "description" : "", - "version" : 1, - "kibanaSavedObjectMeta" : { - "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - }, - "uiStateJSON" : "{}", - "title" : "Total appointments" - }, - "references" : [ - { - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", - "type" : "index-pattern", - "name" : "kibanaSavedObjectMeta.searchSourceJSON.index" - } - ] - }, - "register_requests_visual": { - "attributes" : { - "description" : "", - "version" : 1, - "title" : "register requests", - "visState" : "{\"title\":\"register requests\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"format\":{\"id\":\"number\"},\"params\":{},\"label\":\"Count\",\"aggType\":\"count\"}]}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", - "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#F9934E\"}}}", - "kibanaSavedObjectMeta" : { - "searchSourceJSON" : "{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received register request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received register request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - } - }, - "references" : [ - { - "name" : "kibanaSavedObjectMeta.searchSourceJSON.index", - "type" : "index-pattern", - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09" - }, - { - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", - "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "type" : "index-pattern" - } - ] - }, - "add_appointment_requests_visual": { - "attributes" : { - "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#CCA300\"}}}", - "description" : "", - "visState" : "{\"title\":\"add_appointment requests\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"json\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", - "title" : "add_appointment requests", - "kibanaSavedObjectMeta" : { - "searchSourceJSON" : "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received add_appointment request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received add_appointment request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - }, - "version" : 1 - }, - "references" : [ - { - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", - "type" : "index-pattern", - "name" : "kibanaSavedObjectMeta.searchSourceJSON.index" - }, - { - "name" : "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "id" : "5f36ea30-97e9-11ea-9cf8-038b68181f09", - "type" : "index-pattern" - } - ] - }, - "get_appointment_requests_visual": { - "attributes" : { - "version" : 1, - "uiStateJSON" : "{\"vis\":{\"colors\":{\"Count\":\"#F2C96D\"}}}", - "description" : "", - "kibanaSavedObjectMeta" : { - "searchSourceJSON" : "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":null,\"disabled\":false,\"key\":\"query\",\"negate\":false,\"type\":\"custom\",\"value\":\"{\\\"match\\\":{\\\"doc.message\\\":{\\\"query\\\":\\\"received get_appointment request\\\",\\\"operator\\\":\\\"and\\\"}}}\",\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"doc.message\":{\"operator\":\"and\",\"query\":\"received get_appointment request\"}}}}],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}" - }, - "visState" : "{\"title\":\"get_appointment requests\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"dimensions\":{\"x\":null,\"y\":[{\"accessor\":0,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"label\":\"Count\",\"params\":{}}]},\"grid\":{\"categoryLines\":false},\"labels\":{\"show\":false},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"histogram\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"json\":\"\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"doc.time\",\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}}}]}", - "title" : "get_appointment requests" - }, - "references": [ - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.index", - "type": "index-pattern", - "id": "5f36ea30-97e9-11ea-9cf8-038b68181f09" - }, - { - "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", - "type": "index-pattern", - "id": "5f36ea30-97e9-11ea-9cf8-038b68181f09" - } - ] - } -} - -dashboard = { - "attributes" : { - "version" : 1, - "title" : "Teos System Monitor", - "optionsJSON" : "{\"hidePanelTitles\":false,\"useMargins\":true}", - "panelsJSON" : "[{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"d76b23fc-83b8-49a8-baf4-b1d180d857ca\",\"w\":24,\"x\":0,\"y\":0},\"panelIndex\":\"d76b23fc-83b8-49a8-baf4-b1d180d857ca\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"ce940ff4-87f4-4fe8-b283-c392c08fe0d4\",\"w\":24,\"x\":24,\"y\":0},\"panelIndex\":\"ce940ff4-87f4-4fe8-b283-c392c08fe0d4\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"2b7f4ad4-65b4-42de-af6c-4aaf79d8f4a2\",\"w\":24,\"x\":0,\"y\":30},\"panelIndex\":\"2b7f4ad4-65b4-42de-af6c-4aaf79d8f4a2\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"d7b38be5-9516-4e2b-a86f-5c58f1eb750e\",\"w\":24,\"x\":24,\"y\":15},\"panelIndex\":\"d7b38be5-9516-4e2b-a86f-5c58f1eb750e\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"8ae3cfd9-faa7-4e3e-920a-6a5705b864e9\",\"w\":24,\"x\":0,\"y\":15},\"panelIndex\":\"8ae3cfd9-faa7-4e3e-920a-6a5705b864e9\",\"version\":\"7.6.2\",\"panelRefName\":\"panel_4\"}]", - "hits" : 0, - "kibanaSavedObjectMeta" : { - "searchSourceJSON" : "{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}" - }, - "description" : "", - "timeRestore" : False - } -} diff --git a/monitor/visualizer.py b/monitor/visualizer.py index 957c5b92..89cc45c4 100644 --- a/monitor/visualizer.py +++ b/monitor/visualizer.py @@ -1,11 +1,10 @@ import json +import os import requests from monitor.data_loader import LOG_PREFIX from common.logger import Logger -from monitor.visualizations import index_pattern, visualizations, dashboard - logger = Logger(actor="Visualizer", log_name_prefix=LOG_PREFIX) @@ -13,6 +12,7 @@ class Visualizer: def __init__(self, kibana_host, kibana_port, max_users): self.kibana_endpoint = "http://{}:{}".format(kibana_host, kibana_port) self.saved_obj_endpoint = "{}/api/saved_objects/".format(self.kibana_endpoint) + self.space_endpoint = "{}/api/spaces/space/".format(self.kibana_endpoint) self.headers = headers = { "Content-Type": "application/json", "kbn-xsrf": "true" @@ -20,6 +20,18 @@ def __init__(self, kibana_host, kibana_port, max_users): self.max_users = max_users def create_dashboard(self): + index_pattern = None + visualizations = None + dashboard = None + image_url = None + + with open(os.getcwd() + '/monitor/kibana_data.json') as json_file: + data = json.load(json_file) + index_pattern = data.get("index_pattern") + visualizations = data.get("visualizations") + dashboard = data.get("dashboard") + image_url = data.get("imageUrl") + index_id = None # Find index pattern id if it exists. If it does not, create one to pull Elasticsearch data into Kibana. @@ -66,6 +78,14 @@ def create_dashboard(self): self.create_saved_object("dashboard", dashboard.get("attributes"), visuals) + space_id = "default" + space_name = "system-monitor" + + # Customize the Kibana "space" with Teos logo. + if self.get_space(space_id).get("name") != space_name: + self.customize_space(space_id, space_name, image_url) + + def find(self, obj_type, search_field, search): endpoint = "{}{}".format(self.saved_obj_endpoint, "_find") @@ -119,3 +139,25 @@ def create_saved_object(self, obj_type, attributes, references): logger.info("New Kibana saved object was created") return response.json() + + def get_space(self, space_id): + endpoint = "{}{}".format(self.space_endpoint, space_id) + + response = requests.get(endpoint, headers=self.headers) + + return response.json() + + def customize_space(self, space_id, space_name, image_url): + endpoint = "{}{}".format(self.space_endpoint, space_id) + + data = { + "id": space_id, + "name": space_name, + "imageUrl": image_url + } + + data = json.dumps(data) + + response = requests.put(endpoint, data=data, headers=self.headers) + + return response.json()