diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e66116..2c677862 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Monitoring Plugins: * gitlab-health (fix #670) * gitlab-liveness (fix #670) +* gitlab-readiness (fix #670) * gitlab-version * openstack-nova-list * postgresql-version diff --git a/assets/icingaweb2-module-director/all-the-rest.json b/assets/icingaweb2-module-director/all-the-rest.json index aff3310c..c318a36e 100644 --- a/assets/icingaweb2-module-director/all-the-rest.json +++ b/assets/icingaweb2-module-director/all-the-rest.json @@ -9172,6 +9172,49 @@ "volatile": null, "zone": null }, + "GitLab Readiness": { + "action_url": null, + "apply_for": null, + "assign_filter": null, + "check_command": null, + "check_interval": null, + "check_period": null, + "check_timeout": null, + "command_endpoint": null, + "disabled": false, + "display_name": null, + "enable_active_checks": null, + "enable_event_handler": null, + "enable_flapping": null, + "enable_notifications": null, + "enable_passive_checks": null, + "enable_perfdata": null, + "event_command": null, + "fields": [], + "flapping_threshold_high": null, + "flapping_threshold_low": null, + "groups": [], + "host": null, + "icon_image": null, + "icon_image_alt": null, + "imports": [ + "tpl-service-gitlab-readiness" + ], + "max_check_attempts": null, + "notes": null, + "notes_url": null, + "object_name": "GitLab Readiness", + "object_type": "object", + "retry_interval": null, + "service_set": null, + "template_choice": null, + "use_agent": null, + "use_var_overrides": null, + "uuid": "0e062d6a-e33e-49d2-9709-59450bfce9ac", + "vars": {}, + "volatile": null, + "zone": null + }, "GitLab Version": { "action_url": null, "apply_for": null, diff --git a/check-plugins/gitlab-readiness/README.rst b/check-plugins/gitlab-readiness/README.rst new file mode 100644 index 00000000..b9569168 --- /dev/null +++ b/check-plugins/gitlab-readiness/README.rst @@ -0,0 +1,106 @@ +Check gitlab-readiness +===================== + +Overview +-------- + +The readiness probe checks whether the GitLab instance is ready to accept traffic via Rails Controllers. The check also validates the dependent services (Database, Redis, Gitaly etc.) and gives a status for each. + +Hints: + +* Requires GitLab 9.1.0+ +* To access monitoring resources, the requesting client IP needs to be included in the allowlist. For details, see `how to add IPs to the allowlist for the monitoring endpoints `. +* This check is being exempt from Rack Attack. +* GitLab Health Checks: https://docs.gitlab.com/ee/administration/monitoring/health_check.html + + +Fact Sheet +---------- + +.. csv-table:: + :widths: 30, 70 + + "Check Plugin Download", "https://github.com/Linuxfabrik/monitoring-plugins/tree/main/check-plugins/gitlab-readiness" + "Check Interval Recommendation", "Once a minute" + "Can be called without parameters", "Yes" + "Compiled for", "Linux" + + +Help +---- + +.. code-block:: text + + usage: gitlab-readiness [-h] [-V] [--always-ok] [--severity {warn,crit}] + [--test TEST] [--timeout TIMEOUT] [--url URL] + + The readiness probe checks whether the GitLab instance is ready to accept + traffic via Rails Controllers. The check also validates the dependent services (Database, + Redis, Gitaly etc.) and gives a status for each. + + options: + -h, --help show this help message and exit + -V, --version show program's version number and exit + --always-ok Always returns OK. + --severity {warn,crit} + Severity for alerting. Default: warn + --test TEST For unit tests. Needs "path-to-stdout-file,path-to- + stderr-file,expected-retc". + --timeout TIMEOUT Network timeout in seconds. Default: 3 (seconds) + --url URL GitLab readiness URL endpoint. Default: + http://localhost/-/readiness?all=1 + + +Usage Examples +-------------- + +.. code-block:: bash + + ./gitlab-readiness --severity warn --timeout 3 --url http://localhost/-/readiness + +Output: + +.. code-block:: text + + There are issues with gitaly_check. Run `curl http://localhost/-/readiness?all=1` for full results. + + Service ! Message + ------------------+------------------------------------------------------------- + cache ! Running + chat ! Running + cluster_cache ! Running + db ! Running + db_load_balancing ! Running + feature_flag ! Running + gitaly ! [WARNING] 14:connections to all backends failing; last e... + master ! Running + queues ! Running + rate_limiting ! Running + repository_cache ! Running + sessions ! Running + shared_state ! Running + trace_chunks ! Running + + +States +------ + +* Depending on the given ``--severity``, returns WARN (default) or CRIT if readiness and readiness probes to indicate service health and reachability to required services fail. + + +Perfdata / Metrics +------------------ + +.. csv-table:: + :widths: 25, 15, 60 + :header-rows: 1 + + Name, Type, Description + gitlab-readiness-state, Number, "The current state (0 = OK, 1 = WARN, 2 = CRIT, 3 = UNKNOWN)." + + +Credits, License +---------------- + +* Authors: `Linuxfabrik GmbH, Zurich `_ +* License: The Unlicense, see `LICENSE file `_. diff --git a/check-plugins/gitlab-readiness/gitlab-readiness b/check-plugins/gitlab-readiness/gitlab-readiness new file mode 100755 index 00000000..9ce60f19 --- /dev/null +++ b/check-plugins/gitlab-readiness/gitlab-readiness @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.rst + +"""See the check's README for more details. +""" + +import argparse # pylint: disable=C0413 +import json # pylint: disable=C0413 +import sys # pylint: disable=C0413 + +import lib.args # pylint: disable=C0413 +import lib.base # pylint: disable=C0413 +import lib.test # pylint: disable=C0413 +import lib.url # pylint: disable=C0413 +from lib.globals import (STATE_CRIT, STATE_OK, # pylint: disable=C0413 + STATE_UNKNOWN, STATE_WARN) + + +__author__ = 'Linuxfabrik GmbH, Zurich/Switzerland' +__version__ = '2023082501' + +DESCRIPTION = """The readiness probe checks whether the GitLab instance is ready to accept + traffic via Rails Controllers. + The check also validates the dependent services (Database, Redis, Gitaly etc.) + and gives a status for each.""" + +DEFAULT_SEVERITY = 'warn' +DEFAULT_TIMEOUT = 3 +DEFAULT_URL = 'http://localhost/-/readiness?all=1' + + +def parse_args(): + """Parse command line arguments using argparse. + """ + parser = argparse.ArgumentParser(description=DESCRIPTION) + + parser.add_argument( + '-V', '--version', + action='version', + version='%(prog)s: v{} by {}'.format(__version__, __author__) + ) + + parser.add_argument( + '--always-ok', + help='Always returns OK.', + dest='ALWAYS_OK', + action='store_true', + default=False, + ) + + parser.add_argument( + '--severity', + help='Severity for alerting. Default: %(default)s', + dest='SEVERITY', + default=DEFAULT_SEVERITY, + choices=['warn', 'crit'], + ) + + parser.add_argument( + '--test', + help='For unit tests. Needs "path-to-stdout-file,path-to-stderr-file,expected-retc".', + dest='TEST', + type=lib.args.csv, + ) + + parser.add_argument( + '--timeout', + help='Network timeout in seconds. Default: %(default)s (seconds)', + dest='TIMEOUT', + type=int, + default=DEFAULT_TIMEOUT, + ) + + parser.add_argument( + '--url', + help='GitLab readiness URL endpoint. Default: %(default)s', + dest='URL', + default=DEFAULT_URL, + ) + + return parser.parse_args() + + +def main(): + """The main function. Hier spielt die Musik. + """ + + # parse the command line, exit with UNKNOWN if it fails + try: + args = parse_args() + except SystemExit: + sys.exit(STATE_UNKNOWN) + + if not args.URL.endswith('?all=1'): + args.URL += '?all=1' + + # init some vars + state = STATE_OK + table_data = [] + issues = [] + + # fetch and analyze data + if args.TEST is None: + result = lib.base.coe(lib.url.fetch_json(args.URL, timeout=args.TIMEOUT)) + else: + # do not call the command, put in test data + stdout, stderr, retc = lib.test.test(args.TEST) + result = json.loads(stdout) + + if 'status' in result and 'cache_check' in result and not 'error' in result: + for check in [ + 'cache_check', + 'chat_check', + 'cluster_cache_check', + 'db_check', + 'db_load_balancing_check', + 'feature_flag_check', + 'gitaly_check', + 'master_check', + 'queues_check', + 'rate_limiting_check', + 'repository_cache_check', + 'sessions_check', + 'shared_state_check', + 'trace_chunks_check', + ]: + if result.get(check)[0].get('status', '') != 'ok': + check_state = lib.base.str2state(args.SEVERITY) + state = lib.base.get_worst(state, check_state) + check_msg = lib.base.state2str(check_state, suffix=' ') + \ + result.get(check)[0].get('message', '')[0:46] + '...' + issues.append(check) + else: + check_state = STATE_OK + check_msg = 'Running' + table_data.append({ + 'check': check.replace('_check', ''), + 'msg': check_msg, + }) + if state == STATE_OK: + msg = 'Everything is ok.' + else: + msg = 'There are issues with {}. Run `curl {}` for full results.'.format( + ', '.join(issues), + args.URL, + ) + elif 'status' in result and 'message' in result and not 'error' in result: + msg = result['message'] + state = lib.base.str2state(args.SEVERITY) + elif 'error' in result: + msg = result['error'] + state = lib.base.str2state(args.SEVERITY) + else: + msg = 'Unknown error fetching URL {}'.format(args.URL) + state = STATE_UNKNOWN + perfdata = lib.base.get_perfdata('gitlab-readiness', state, None, None, None, 0, STATE_UNKNOWN) + + # build the message + if table_data: + msg += '\n\n' + lib.base.get_table( + table_data, + ['check', 'msg'], + header=['Service', 'Message'], + ) + + # over and out + lib.base.oao(msg, state, perfdata, always_ok=args.ALWAYS_OK) + + +if __name__ == '__main__': + try: + main() + except Exception: # pylint: disable=W0703 + lib.base.cu() diff --git a/check-plugins/gitlab-readiness/grafana/gitlab-readiness.yml b/check-plugins/gitlab-readiness/grafana/gitlab-readiness.yml new file mode 100644 index 00000000..2aa7493a --- /dev/null +++ b/check-plugins/gitlab-readiness/grafana/gitlab-readiness.yml @@ -0,0 +1,115 @@ +apiVersion: grizzly.grafana.com/v1alpha1 +kind: Dashboard +metadata: + folder: linuxfabrik-monitoring-plugins + name: gitlab-readiness +spec: + schemaVersion: 2023041201 + tags: + - Linuxfabrik + - Grizzly + - static + time: + from: now-90d + to: now + timepicker: + hidden: false + refresh_intervals: + - 1m + timezone: browser + title: GitLab Readiness + uid: linuxfabrik-monitoring-plugins-gitlab-readiness + editable: true + liveNow: true + refresh: 1m + templating: + list: + - hide: 2 + label: Command + name: command + query: cmd-check-gitlab-readiness + type: constant + - label: Hostname + name: hostname + query: SHOW TAG VALUES FROM "cmd-check-gitlab-readiness" WITH KEY = "hostname" + refresh: 2 + sort: 1 + type: query + + panels: + + - title: GitLab Readiness + type: timeseries + gridPos: + h: 8 + w: 12 + x: 12 + y: 8 + fieldConfig: + defaults: + color: + mode: palette-classic + custom: + lineInterpolation: smooth + spanNulls: true + decimals: 0 + max: 3 + min: 0 + unit: short + overrides: + - matcher: + id: byName + options: gitlab-readiness + properties: + - id: mappings + value: + - options: + '0': + text: OK + '1': + text: WARN + '2': + text: CRIT + '3': + text: UNKN + type: value + options: + legend: + calcs: + - min + - max + displayMode: table + placement: bottom + showLegend: true + tooltip: + mode: multi + sort: none + + targets: + + - alias: gitlab-readiness + refId: gitlab-readiness + groupBy: + - params: + - $interval + type: time + measurement: /^$command$/ + resultFormat: time_series + select: + - - params: + - value + type: field + - params: [] + type: mean + tags: + - key: hostname + operator: '=~' + value: /^$hostname$/ + - condition: AND + key: service + operator: '=' + value: GitLab Readiness + - condition: AND + key: metric + operator: '=' + value: gitlab-readiness diff --git a/check-plugins/gitlab-readiness/icingaweb2-module-director/gitlab-readiness.json b/check-plugins/gitlab-readiness/icingaweb2-module-director/gitlab-readiness.json new file mode 100644 index 00000000..bea45be5 --- /dev/null +++ b/check-plugins/gitlab-readiness/icingaweb2-module-director/gitlab-readiness.json @@ -0,0 +1,171 @@ +{ + "Command": { + "cmd-check-gitlab-readiness": { + "arguments": { + "--always-ok": { + "set_if": "$gitlab_readiness_always_ok$" + }, + "--severity": { + "value": "$gitlab_readiness_severity$" + }, + "--timeout": { + "value": "$gitlab_readiness_timeout$" + }, + "--url": { + "value": "$gitlab_readiness_url$" + } + }, + "command": "/usr/lib64/nagios/plugins/gitlab-readiness", + "disabled": false, + "fields": [ + { + "datafield_id": 1, + "is_required": "n", + "var_filter": null + }, + { + "datafield_id": 2, + "is_required": "n", + "var_filter": null + }, + { + "datafield_id": 3, + "is_required": "n", + "var_filter": null + }, + { + "datafield_id": 4, + "is_required": "n", + "var_filter": null + } + ], + "imports": [], + "is_string": null, + "methods_execute": "PluginCheck", + "object_name": "cmd-check-gitlab-readiness", + "object_type": "object", + "timeout": "10", + "vars": {}, + "zone": null, + "uuid": "a354fe07-e065-459c-87fe-9f5829cf61fe" + } + }, + "ServiceTemplate": { + "tpl-service-gitlab-readiness": { + "action_url": null, + "apply_for": null, + "assign_filter": null, + "check_command": "cmd-check-gitlab-readiness", + "check_interval": 60, + "check_period": null, + "check_timeout": null, + "command_endpoint": null, + "disabled": false, + "display_name": null, + "enable_active_checks": null, + "enable_event_handler": null, + "enable_flapping": null, + "enable_notifications": true, + "enable_passive_checks": null, + "enable_perfdata": true, + "event_command": null, + "fields": [], + "flapping_threshold_high": null, + "flapping_threshold_low": null, + "groups": [], + "host": null, + "icon_image": "gitlab-readiness.png", + "icon_image_alt": null, + "imports": [ + "tpl-service-generic" + ], + "max_check_attempts": 5, + "notes": "The readiness probe checks whether the GitLab instance is ready to accept traffic via Rails Controllers. The check also validates the dependent services (Database, Redis, Gitaly etc.) and gives a status for each.", + "notes_url": "https://github.com/Linuxfabrik/monitoring-plugins/tree/main/check-plugins/gitlab-readiness", + "object_name": "tpl-service-gitlab-readiness", + "object_type": "template", + "retry_interval": 15, + "service_set": null, + "template_choice": null, + "use_agent": null, + "use_var_overrides": null, + "vars": { + "criticality": "C", + "gitlab_readiness_always_ok": false, + "gitlab_readiness_severity": "warn", + "gitlab_readiness_timeout": 3, + "gitlab_readiness_url": "http://localhost/-/readiness?all=1" + }, + "volatile": null, + "zone": null, + "uuid": "183dae6f-2dab-4d15-898d-90bb52e6b5fc" + } + }, + "DataList": { + "gitlab_readiness_severity_list": { + "list_name": "gitlab_readiness_severity_list", + "owner": "icinga-admin", + "entries": [ + { + "entry_name": "warn", + "entry_value": "Warn", + "format": "string", + "allowed_roles": null + }, + { + "entry_name": "crit", + "entry_value": "Crit", + "format": "string", + "allowed_roles": null + } + ], + "uuid": "d506967e-7834-4ebb-937d-7d1855f58562" + } + }, + "Datafield": { + "1": { + "varname": "gitlab_readiness_always_ok", + "caption": "Gitlab Readiness: Always OK?", + "description": "Always returns OK.", + "datatype": "Icinga\\Module\\Director\\DataType\\DataTypeBoolean", + "format": null, + "settings": {}, + "uuid": "3d4c677b-e565-4dff-8d34-c0c57445378e" + }, + "2": { + "varname": "gitlab_readiness_severity", + "caption": "Gitlab Readiness: Severity", + "description": "Severity for alerting.", + "datatype": "Icinga\\Module\\Director\\DataType\\DataTypeDatalist", + "format": null, + "settings": { + "behavior": "strict", + "data_type": "string", + "datalist": "gitlab_readiness_severity_list" + }, + "uuid": "46d90e23-16b7-4ba0-9e19-714281ce4d51" + }, + "3": { + "varname": "gitlab_readiness_timeout", + "caption": "Gitlab Readiness: Timeout", + "description": "Network timeout in seconds.", + "datatype": "Icinga\\Module\\Director\\DataType\\DataTypeString", + "format": null, + "settings": { + "visibility": "visible" + }, + "uuid": "37042d91-c602-457f-9a66-66e899aba7a7" + }, + "4": { + "varname": "gitlab_readiness_url", + "caption": "Gitlab Readiness: URL", + "description": "GitLab readiness URL endpoint.", + "datatype": "Icinga\\Module\\Director\\DataType\\DataTypeString", + "format": null, + "settings": { + "visibility": "visible" + }, + "uuid": "8a1a74fc-1d09-413e-9a6d-219f30682831" + } + } +} diff --git a/check-plugins/gitlab-readiness/icingaweb2-module-director/gitlab-readiness.yml b/check-plugins/gitlab-readiness/icingaweb2-module-director/gitlab-readiness.yml new file mode 100644 index 00000000..30db3571 --- /dev/null +++ b/check-plugins/gitlab-readiness/icingaweb2-module-director/gitlab-readiness.yml @@ -0,0 +1,6 @@ +--- +variants: + - linux + +overwrites: + '["ServiceTemplate"]["tpl-service-gitlab-readiness"]["enable_perfdata"]': true diff --git a/check-plugins/gitlab-readiness/icingaweb2-module-grafana/gitlab-readiness.ini b/check-plugins/gitlab-readiness/icingaweb2-module-grafana/gitlab-readiness.ini new file mode 100644 index 00000000..7281af04 --- /dev/null +++ b/check-plugins/gitlab-readiness/icingaweb2-module-grafana/gitlab-readiness.ini @@ -0,0 +1,6 @@ +[cmd-check-gitlab-readiness] +dashboard = "GitLab Readiness" +panelId = "1" +orgId = "" +repeatable = "no" +dashboarduid = "gitlab-readiness" diff --git a/check-plugins/gitlab-readiness/lib b/check-plugins/gitlab-readiness/lib new file mode 120000 index 00000000..58677ddb --- /dev/null +++ b/check-plugins/gitlab-readiness/lib @@ -0,0 +1 @@ +../../lib \ No newline at end of file diff --git a/check-plugins/gitlab-readiness/unit-test/run b/check-plugins/gitlab-readiness/unit-test/run new file mode 100755 index 00000000..cfd78aaf --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/run @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8; py-indent-offset: 4 -*- +# +# Author: Linuxfabrik GmbH, Zurich, Switzerland +# Contact: info (at) linuxfabrik (dot) ch +# https://www.linuxfabrik.ch/ +# License: The Unlicense, see LICENSE file. + +# https://github.com/Linuxfabrik/monitoring-plugins/blob/main/CONTRIBUTING.rst + +import sys +sys.path.append("..") # Adds higher directory to python modules path. + + + +import unittest + +from lib.globals import STATE_OK, STATE_UNKNOWN, STATE_WARN, STATE_CRIT +import lib.base +import lib.shell + + +class TestCheck(unittest.TestCase): + + check = '../gitlab-readiness' + + #self.assertEqual(retc, STATE_CRIT) + #self.assertIn('Waiting for more data (1).', stdout) + #self.assertRegex(stdout, r'1 error in Kernel Ring Buffer.') + + def test_if_check_runs_EXAMPLE01(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE01,,0')) + self.assertIn('Everything is ok.', stdout) + self.assertIn('Service ! Message ', stdout) + self.assertIn('------------------+---------', stdout) + self.assertIn('cache ! Running ', stdout) + self.assertIn('chat ! Running ', stdout) + self.assertIn('cluster_cache ! Running ', stdout) + self.assertIn('db ! Running ', stdout) + self.assertIn('db_load_balancing ! Running ', stdout) + self.assertIn('feature_flag ! Running ', stdout) + self.assertIn('gitaly ! Running ', stdout) + self.assertIn('master ! Running ', stdout) + self.assertIn('queues ! Running ', stdout) + self.assertIn('rate_limiting ! Running ', stdout) + self.assertIn('repository_cache ! Running ', stdout) + self.assertIn('sessions ! Running ', stdout) + self.assertIn('shared_state ! Running ', stdout) + self.assertIn('trace_chunks ! Running', stdout) + self.assertEqual(retc, STATE_OK) + + def test_if_check_runs_EXAMPLE02(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE02,,0')) + self.assertIn('There are issues with cache_check, chat_check, cluster_cache_check, db_load_balancing_check, feature_flag_check, queues_check, rate_limiting_check, repository_cache_check, sessions_check, shared_state_check, trace_chunks_check. Run `curl http://localhost/-/readiness?all=1` for full results.', stdout) + self.assertIn('Service ! Message ', stdout) + self.assertIn('------------------+-------------------------------------------------------------', stdout) + self.assertIn('cache ! [WARNING] unexpected Cache check result: Error connectin... ', stdout) + self.assertIn('chat ! [WARNING] unexpected Chat check result: Error connecting... ', stdout) + self.assertIn('cluster_cache ! [WARNING] unexpected Cluster_cache check result: Error c... ', stdout) + self.assertIn('db ! Running ', stdout) + self.assertIn('db_load_balancing ! [WARNING] unexpected Db_load_balancing check result: Err... ', stdout) + self.assertIn('feature_flag ! [WARNING] unexpected Feature_flag check result: Error co... ', stdout) + self.assertIn('gitaly ! Running ', stdout) + self.assertIn('master ! Running ', stdout) + self.assertIn('queues ! [WARNING] unexpected Queues check result: Error connecti... ', stdout) + self.assertIn('rate_limiting ! [WARNING] unexpected Rate_limiting check result: Error c... ', stdout) + self.assertIn('repository_cache ! [WARNING] unexpected Repository_cache check result: Erro... ', stdout) + self.assertIn('sessions ! [WARNING] unexpected Sessions check result: Error connec... ', stdout) + self.assertIn('shared_state ! [WARNING] unexpected Shared_state check result: Error co... ', stdout) + self.assertIn('trace_chunks ! [WARNING] unexpected Trace_chunks check result: Error co...|', stdout) + + def test_if_check_runs_EXAMPLE03(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE03,,0')) + self.assertIn('There are issues with gitaly_check. Run `curl http://localhost/-/readiness?all=1` for full results.', stdout) + self.assertIn('Service ! Message ', stdout) + self.assertIn('------------------+-------------------------------------------------------------', stdout) + self.assertIn('cache ! Running ', stdout) + self.assertIn('chat ! Running ', stdout) + self.assertIn('cluster_cache ! Running ', stdout) + self.assertIn('db ! Running ', stdout) + self.assertIn('db_load_balancing ! Running ', stdout) + self.assertIn('feature_flag ! Running ', stdout) + self.assertIn('gitaly ! [WARNING] 14:connections to all backends failing; last e... ', stdout) + self.assertIn('master ! Running ', stdout) + self.assertIn('queues ! Running ', stdout) + self.assertIn('rate_limiting ! Running ', stdout) + self.assertIn('repository_cache ! Running ', stdout) + self.assertIn('sessions ! Running ', stdout) + self.assertIn('shared_state ! Running ', stdout) + self.assertIn('trace_chunks ! Running', stdout) + self.assertEqual(retc, STATE_WARN) + + def test_if_check_runs_EXAMPLE04(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE04,,0')) + self.assertIn('There are issues with gitaly_check. Run `curl http://localhost/-/readiness?all=1` for full results.', stdout) + self.assertIn('Service ! Message ', stdout) + self.assertIn('------------------+-------------------------------------------------------------', stdout) + self.assertIn('cache ! Running ', stdout) + self.assertIn('chat ! Running ', stdout) + self.assertIn('cluster_cache ! Running ', stdout) + self.assertIn('db ! Running ', stdout) + self.assertIn('db_load_balancing ! Running ', stdout) + self.assertIn('feature_flag ! Running ', stdout) + self.assertIn('gitaly ! [WARNING] 14:connections to all backends failing; last e... ', stdout) + self.assertIn('master ! Running ', stdout) + self.assertIn('queues ! Running ', stdout) + self.assertIn('rate_limiting ! Running ', stdout) + self.assertIn('repository_cache ! Running ', stdout) + self.assertIn('sessions ! Running ', stdout) + self.assertIn('shared_state ! Running ', stdout) + self.assertIn('trace_chunks ! Running', stdout) + self.assertEqual(retc, STATE_WARN) + + def test_if_check_runs_EXAMPLE05(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE05,,0')) + self.assertIn('ActiveRecord::DatabaseConnectionError : There is an issue connecting to your database with your username/password, username: gitlab.\n\nPlease check your database configuration to ensure the username/password are valid.', stdout) + self.assertEqual(stderr, '') + self.assertEqual(retc, STATE_WARN) + + def test_if_check_runs_EXAMPLE06(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE06,,0')) + self.assertIn('Bad Gateway', stdout) + self.assertEqual(stderr, '') + self.assertEqual(retc, STATE_WARN) + + def test_if_check_runs_EXAMPLE07(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE07,,0')) + self.assertIn('nginx returned 502 when communicating with gitlab-workhorse', stdout) + self.assertEqual(stderr, '') + self.assertEqual(retc, STATE_WARN) + + def test_if_check_runs_EXAMPLE08(self): + stdout, stderr, retc = lib.base.coe(lib.shell.shell_exec(self.check + ' --test=stdout/EXAMPLE08,,0')) + self.assertIn('Unknown error fetching URL http://localhost/-/readiness', stdout) + self.assertEqual(stderr, '') + self.assertEqual(retc, STATE_UNKNOWN) + + +if __name__ == '__main__': + unittest.main() diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE01 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE01 new file mode 100644 index 00000000..989bb58c --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE01 @@ -0,0 +1,76 @@ +{ + "status": "ok", + "master_check": [ + { + "status": "ok" + } + ], + "db_check": [ + { + "status": "ok" + } + ], + "cache_check": [ + { + "status": "ok" + } + ], + "cluster_cache_check": [ + { + "status": "ok" + } + ], + "db_load_balancing_check": [ + { + "status": "ok" + } + ], + "feature_flag_check": [ + { + "status": "ok" + } + ], + "queues_check": [ + { + "status": "ok" + } + ], + "rate_limiting_check": [ + { + "status": "ok" + } + ], + "repository_cache_check": [ + { + "status": "ok" + } + ], + "sessions_check": [ + { + "status": "ok" + } + ], + "shared_state_check": [ + { + "status": "ok" + } + ], + "trace_chunks_check": [ + { + "status": "ok" + } + ], + "chat_check": [ + { + "status": "ok" + } + ], + "gitaly_check": [ + { + "status": "ok", + "labels": { + "shard": "default" + } + } + ] +} diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE02 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE02 new file mode 100644 index 00000000..4443f40c --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE02 @@ -0,0 +1,87 @@ +{ + "status": "failed", + "master_check": [ + { + "status": "ok" + } + ], + "db_check": [ + { + "status": "ok" + } + ], + "cache_check": [ + { + "status": "failed", + "message": "unexpected Cache check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "cluster_cache_check": [ + { + "status": "failed", + "message": "unexpected Cluster_cache check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "db_load_balancing_check": [ + { + "status": "failed", + "message": "unexpected Db_load_balancing check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "feature_flag_check": [ + { + "status": "failed", + "message": "unexpected Feature_flag check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "queues_check": [ + { + "status": "failed", + "message": "unexpected Queues check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "rate_limiting_check": [ + { + "status": "failed", + "message": "unexpected Rate_limiting check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "repository_cache_check": [ + { + "status": "failed", + "message": "unexpected Repository_cache check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "sessions_check": [ + { + "status": "failed", + "message": "unexpected Sessions check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "shared_state_check": [ + { + "status": "failed", + "message": "unexpected Shared_state check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "trace_chunks_check": [ + { + "status": "failed", + "message": "unexpected Trace_chunks check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "chat_check": [ + { + "status": "failed", + "message": "unexpected Chat check result: Error connecting to Redis on /var/opt/gitlab/redis/redis.socket (Errno::ENOENT)" + } + ], + "gitaly_check": [ + { + "status": "ok", + "labels": { + "shard": "default" + } + } + ] +} diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE03 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE03 new file mode 100644 index 00000000..371e05b4 --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE03 @@ -0,0 +1,77 @@ +{ + "status": "failed", + "master_check": [ + { + "status": "ok" + } + ], + "db_check": [ + { + "status": "ok" + } + ], + "cache_check": [ + { + "status": "ok" + } + ], + "cluster_cache_check": [ + { + "status": "ok" + } + ], + "db_load_balancing_check": [ + { + "status": "ok" + } + ], + "feature_flag_check": [ + { + "status": "ok" + } + ], + "queues_check": [ + { + "status": "ok" + } + ], + "rate_limiting_check": [ + { + "status": "ok" + } + ], + "repository_cache_check": [ + { + "status": "ok" + } + ], + "sessions_check": [ + { + "status": "ok" + } + ], + "shared_state_check": [ + { + "status": "ok" + } + ], + "trace_chunks_check": [ + { + "status": "ok" + } + ], + "chat_check": [ + { + "status": "ok" + } + ], + "gitaly_check": [ + { + "status": "failed", + "message": "14:connections to all backends failing; last error: UNKNOWN: unix:/var/opt/gitlab/gitaly/gitaly.socket: No such file or directory. debug_error_string:{UNKNOWN:connections to all backends failing; last error: UNKNOWN: unix:/var/opt/gitlab/gitaly/gitaly.socket: No such file or directory {grpc_status:14, created_time:\"2023-08-25T13:00:00.000000000+02:00\"}}", + "labels": { + "shard": "default" + } + } + ] +} diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE04 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE04 new file mode 100644 index 00000000..f5fb2a03 --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE04 @@ -0,0 +1,77 @@ +{ + "status": "failed", + "master_check": [ + { + "status": "ok" + } + ], + "db_check": [ + { + "status": "ok" + } + ], + "cache_check": [ + { + "status": "ok" + } + ], + "cluster_cache_check": [ + { + "status": "ok" + } + ], + "db_load_balancing_check": [ + { + "status": "ok" + } + ], + "feature_flag_check": [ + { + "status": "ok" + } + ], + "queues_check": [ + { + "status": "ok" + } + ], + "rate_limiting_check": [ + { + "status": "ok" + } + ], + "repository_cache_check": [ + { + "status": "ok" + } + ], + "sessions_check": [ + { + "status": "ok" + } + ], + "shared_state_check": [ + { + "status": "ok" + } + ], + "trace_chunks_check": [ + { + "status": "ok" + } + ], + "chat_check": [ + { + "status": "ok" + } + ], + "gitaly_check": [ + { + "status": "failed", + "message": "14:connections to all backends failing; last error: UNKNOWN: unix:/var/opt/gitlab/gitaly/gitaly.socket: No such file or directory. debug_error_string:{UNKNOWN:connections to all backends failing; last error: UNKNOWN: unix:/var/opt/gitlab/gitaly/gitaly.socket: No such file or directory {created_time:\"2023-08-25T13:00:00.000000000+02:00\", grpc_status:14}}", + "labels": { + "shard": "default" + } + } + ] +} diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE05 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE05 new file mode 100644 index 00000000..17aa28f1 --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE05 @@ -0,0 +1,4 @@ +{ + "status": "failed", + "message": "ActiveRecord::DatabaseConnectionError : There is an issue connecting to your database with your username/password, username: gitlab.\n\nPlease check your database configuration to ensure the username/password are valid.\n" +} diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE06 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE06 new file mode 100644 index 00000000..578460a9 --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE06 @@ -0,0 +1 @@ +{"error":"Bad Gateway","status":502} \ No newline at end of file diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE07 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE07 new file mode 100644 index 00000000..4fcdc3b2 --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE07 @@ -0,0 +1 @@ +{"error":"nginx returned 502 when communicating with gitlab-workhorse","status":502} \ No newline at end of file diff --git a/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE08 b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE08 new file mode 100644 index 00000000..893474f2 --- /dev/null +++ b/check-plugins/gitlab-readiness/unit-test/stdout/EXAMPLE08 @@ -0,0 +1 @@ +{"status":"ok","master_check":[{"status":"ok"}]} \ No newline at end of file