diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java index d5dab7800c41..b746f7eaea18 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/placement/metrics/SCMMetrics.java @@ -180,7 +180,9 @@ public void addRatisEvent(String event) { } } - @Metric("Ratis state machine events") + // Ratis state machine events are multi-line logs, which should not be + // published as time-series metrics to metrics systems like Prometheus. + // Instead, they are exposed via JMX / MXBean endpoints. public String getRatisEvents() { synchronized (ratisEvents) { return String.join("\n", ratisEvents); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java index 947484864e40..0fcc4625387c 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMMXBean.java @@ -82,4 +82,6 @@ public interface SCMMXBean extends ServiceRuntimeInfo { * @return the SCM hostname for the datanode. */ String getHostname(); + + String getRatisEvents(); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 59a04d6a9a54..13e297b9caed 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@ -2243,6 +2243,11 @@ public String getHostname() { return scmHostName; } + @Override + public String getRatisEvents() { + return metrics != null ? metrics.getRatisEvents() : ""; + } + public Collection getScmAdminUsernames() { return scmAdmins.getAdminUsernames(); } diff --git a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js index 27ecc7f8155c..73ecda3d49c3 100644 --- a/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js +++ b/hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm.js @@ -30,10 +30,10 @@ templateUrl: 'ratis-events.html', controller: function ($http) { var ctrl = this; - $http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=SCMMetrics") + $http.get("jmx?qry=Hadoop:service=StorageContainerManager,name=StorageContainerManagerInfo,component=ServerRuntime") .then(function (result) { var metrics = result.data.beans[0]; - var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : []; + var rawEvents = (metrics && metrics['RatisEvents']) ? metrics['RatisEvents'].split('\n') : []; ctrl.events = rawEvents.map(function(e) { var parts = e.split('|'); return { diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh b/hadoop-ozone/dist/src/main/compose/ozone/test.sh index 800f1c41f281..980d1487f804 100755 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@ -24,6 +24,7 @@ export COMPOSE_DIR export SECURITY_ENABLED=false export OZONE_REPLICATION_FACTOR=3 +export COMPOSE_FILE=docker-compose.yaml:monitoring.yaml # shellcheck source=/dev/null source "$COMPOSE_DIR/../testlib.sh" @@ -40,6 +41,7 @@ execute_robot_test scm gdpr execute_robot_test scm security/ozone-secure-token.robot execute_robot_test scm recon +execute_robot_test scm prometheus execute_robot_test scm om-ratis diff --git a/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot new file mode 100644 index 000000000000..11cc7d6031a3 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus.robot @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Test Prometheus monitoring integration +Library OperatingSystem +Library BuiltIn +Resource ../commonlib.robot + +*** Test Cases *** +Verify Prometheus targets are healthy + Wait Until Keyword Succeeds 90sec 10sec Check Prometheus Targets Health + +*** Keywords *** +Check Prometheus Targets Health + ${result} = Execute python3 ${OZONE_DIR}/smoketest/prometheus/prometheus_check.py + Should Contain ${result} Successfully verified diff --git a/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py new file mode 100644 index 000000000000..6fe85be8341c --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/prometheus/prometheus_check.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import urllib.request +import json +import sys +import socket + +def is_running(host, port): + try: + with socket.create_connection((host, int(port)), timeout=2): + return True + except Exception: + return False + +def main(): + try: + res = urllib.request.urlopen("http://prometheus:9090/api/v1/targets") + data = json.loads(res.read().decode()) + targets = data.get("data", {}).get("activeTargets", []) + if not targets: + print("No active targets found in Prometheus") + sys.exit(1) + + failed = False + checked = 0 + for t in targets: + url = t.get("scrapeUrl", "") + # scrapeUrl is like "http://scm:9876/prom" + try: + host_port = url.split("//")[1].split("/")[0] + if ":" in host_port: + host, port = host_port.split(":") + else: + host = host_port + port = 80 + except Exception: + continue + + if is_running(host, port): + checked += 1 + health = t.get("health", "") + print(f"Target {host}:{port} is running. Prometheus health: {health}") + if health != "up": + print(f"Error: Target {host}:{port} is running but Prometheus health is '{health}'. Last error: {t.get('lastError')}") + failed = True + else: + print(f"Target {host}:{port} is not running. Skipping check.") + + if checked == 0: + print("Error: No running targets were checked!") + sys.exit(1) + + if failed: + sys.exit(1) + + print(f"Successfully verified {checked} running targets.") + except Exception as e: + print(f"Exception during health check: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java index bde82b2ab951..f2ae925dcbdb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSCMMXBean.java @@ -86,6 +86,9 @@ public void testSCMMXBean() throws Exception { double containerThreshold = (double) mbs.getAttribute(bean, "SafeModeCurrentContainerThreshold"); assertEquals(scm.getCurrentContainerThreshold(), containerThreshold, 0); + + String ratisEvents = (String) mbs.getAttribute(bean, "RatisEvents"); + assertEquals(scm.getMetrics().getRatisEvents(), ratisEvents); } @Test diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java index 0e7488ae191f..3d418ef8c839 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMXBean.java @@ -41,4 +41,6 @@ public interface OMMXBean extends ServiceRuntimeInfo { * @return the OM hostname for the datanode. */ String getHostname(); + + String getRatisEvents(); } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java index dc67f3c00d52..7725f7835acb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMMetrics.java @@ -1627,7 +1627,9 @@ public void addRatisEvent(String event) { } } - @Metric("Ratis state machine events") + // Ratis state machine events are multi-line logs, which should not be + // published as time-series metrics to metrics systems like Prometheus. + // Instead, they are exposed via JMX / MXBean endpoints. public String getRatisEvents() { synchronized (ratisEvents) { return String.join("\n", ratisEvents); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index db7717c81ea2..1910d92e9690 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3309,6 +3309,11 @@ public String getHostname() { return omHostName; } + @Override + public String getRatisEvents() { + return metrics != null ? metrics.getRatisEvents() : ""; + } + @VisibleForTesting public OzoneManagerHttpServer getHttpServer() { return httpServer; diff --git a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js index 313c79a3f081..2f501fc98d5b 100644 --- a/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js +++ b/hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/ozoneManager.js @@ -173,10 +173,10 @@ templateUrl: 'ratis-events.html', controller: function ($http) { var ctrl = this; - $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OMMetrics") + $http.get("jmx?qry=Hadoop:service=OzoneManager,name=OzoneManagerInfo,component=ServerRuntime") .then(function (result) { var metrics = result.data.beans[0]; - var rawEvents = metrics['tag.RatisEvents'] ? metrics['tag.RatisEvents'].split('\n') : []; + var rawEvents = (metrics && metrics['RatisEvents']) ? metrics['RatisEvents'].split('\n') : []; ctrl.events = rawEvents.map(function(e) { var parts = e.split('|'); return {