From 3c665eb0afbaa4d6ef73ae63b76110e1cbd1eb4a Mon Sep 17 00:00:00 2001 From: arunk-kumar Date: Fri, 31 Jul 2026 21:33:11 +0530 Subject: [PATCH 1/3] HDDS-16019. Publish httpfs gateway metrics to Prometheus via /prom endpoint Extract the Prometheus servlet wiring from BaseHttpServer into a public static helper (addPrometheusEndpoint) so HttpFSServerWebServer can expose /prom without extending BaseHttpServer. Wire the helper and the DefaultMetricsSystem lifecycle in HttpFSServerWebServer, gated by hdds.prometheus.endpoint.enabled. --- .../hdds/server/http/BaseHttpServer.java | 42 ++++++++++--------- .../fs/http/server/HttpFSServerWebServer.java | 16 +++++++ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java index 9b2a0a61fdac..a00f71f58eaf 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java @@ -155,26 +155,7 @@ public BaseHttpServer(MutableConfigurationSource conf, String name) conf.getBoolean(HddsConfigKeys.HDDS_PROFILER_ENABLED, false); if (prometheusSupport) { - prometheusMetricsSink = new PrometheusMetricsSink(name); - httpServer.getWebAppContext().getServletContext() - .setAttribute(PROMETHEUS_SINK, prometheusMetricsSink); - HddsPrometheusConfig prometheusConfig = - conf.getObject(HddsPrometheusConfig.class); - String token = prometheusConfig.getPrometheusEndpointToken(); - if (StringUtils.isNotEmpty(token)) { - httpServer.getWebAppContext().getServletContext() - .setAttribute(PrometheusServlet.SECURITY_TOKEN, token); - // Adding as internal servlet since we want to have token based - // auth and hence SPNEGO should be disabled if security is enabled. - httpServer.addInternalServlet("prometheus", "/prom", - PrometheusServlet.class); - } else { - // If token is not configured, keeping as regular servlet and not - // internal servlet since we do not want to expose /prom endpoint - // without authentication in a secure cluster. - httpServer.addServlet("prometheus", "/prom", - PrometheusServlet.class); - } + prometheusMetricsSink = addPrometheusEndpoint(httpServer, conf, name); } if (profilerSupport) { @@ -243,6 +224,27 @@ public static HttpServer2.Builder newHttpServer2BuilderForOzone( return builder; } + /** + * Add the Prometheus endpoint at {@code /prom} to an existing {@link HttpServer2} instance. + * Returns the registered sink for use in start/stop lifecycle calls. + */ + public static PrometheusMetricsSink addPrometheusEndpoint( + HttpServer2 httpServer, ConfigurationSource conf, String name) { + PrometheusMetricsSink sink = new PrometheusMetricsSink(name); + httpServer.getWebAppContext().getServletContext().setAttribute(PROMETHEUS_SINK, sink); + String token = conf.getObject(HddsPrometheusConfig.class).getPrometheusEndpointToken(); + if (StringUtils.isNotEmpty(token)) { + httpServer.getWebAppContext().getServletContext() + .setAttribute(PrometheusServlet.SECURITY_TOKEN, token); + // Token-based auth: use internal servlet so SPNEGO is bypassed. + httpServer.addInternalServlet("prometheus", "/prom", PrometheusServlet.class); + } else { + // No token: regular servlet, protected by the server's auth filter in secure mode. + httpServer.addServlet("prometheus", "/prom", PrometheusServlet.class); + } + return sink; + } + /** * Add a servlet to BaseHttpServer. * diff --git a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java index 1523c60cdb77..ac7c4e39d382 100644 --- a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java +++ b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java @@ -28,10 +28,14 @@ import java.util.LinkedHashSet; import java.util.Set; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.server.http.BaseHttpServer; import org.apache.hadoop.hdds.server.http.HttpServer2; +import org.apache.hadoop.hdds.server.http.PrometheusMetricsSink; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.security.authentication.server.ProxyUserAuthenticationFilterInitializer; import org.apache.hadoop.security.authorize.AccessControlList; @@ -72,6 +76,7 @@ public class HttpFSServerWebServer { private final HttpServer2 httpServer; private final String scheme; + private PrometheusMetricsSink prometheusMetricsSink; HttpFSServerWebServer(OzoneConfiguration conf, Configuration sslConf) throws Exception { @@ -129,6 +134,10 @@ public class HttpFSServerWebServer { .setACL(new AccessControlList(conf.get(HTTP_ADMINS_KEY, " "))) .addEndpoint(endpoint) .build(); + + if (conf.getBoolean(HddsConfigKeys.HDDS_PROMETHEUS_ENABLED, true)) { + prometheusMetricsSink = BaseHttpServer.addPrometheusEndpoint(httpServer, conf, NAME); + } } /** @@ -153,6 +162,10 @@ private static void deprecateEnv(String varName, Configuration conf, public void start() throws IOException { httpServer.start(); + if (prometheusMetricsSink != null) { + DefaultMetricsSystem.instance() + .register("prometheus", "Hadoop metrics prometheus exporter", prometheusMetricsSink); + } } public void join() throws InterruptedException { @@ -161,6 +174,9 @@ public void join() throws InterruptedException { public void stop() throws Exception { httpServer.stop(); + if (prometheusMetricsSink != null) { + DefaultMetricsSystem.instance().unregisterSource("prometheus"); + } } public URL getUrl() { From 7fe198d77365325b7a2003c8af1662a0bcf18b96 Mon Sep 17 00:00:00 2001 From: arunk-kumar Date: Sun, 2 Aug 2026 01:55:17 +0530 Subject: [PATCH 2/3] HDDS-16019. Add httpfs to compose Prometheus scrape config Add httpfs:14000 to prometheus.yml scrape targets and add the httpfs service to the monitoring-config anchor in monitoring.yaml, enabling Prometheus to scrape the /prom endpoint added in the previous commit. --- hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml | 2 ++ hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml index 0afae352e342..4c0c83027050 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml +++ b/hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml @@ -52,3 +52,5 @@ services: <<: *monitoring-config recon: <<: *monitoring-config + httpfs: + <<: *monitoring-config diff --git a/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml b/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml index 562bcb9e71ba..eba0575d590b 100644 --- a/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml +++ b/hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml @@ -61,3 +61,12 @@ scrape_configs: - "recon:9888" labels: component: recon + - job_name: ozone-httpfs + metrics_path: /prom + params: + user.name: [hadoop] + static_configs: + - targets: + - "httpfs:14000" + labels: + component: httpfs From dc5ed5ac16e887294229b220833c890c166d9641 Mon Sep 17 00:00:00 2001 From: arunk-kumar Date: Mon, 10 Aug 2026 13:21:20 +0530 Subject: [PATCH 3/3] HDDS-16019. Remove incorrect unregisterSource call from stop() --- .../org/apache/ozone/fs/http/server/HttpFSServerWebServer.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java index ac7c4e39d382..508987d05b9f 100644 --- a/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java +++ b/hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java @@ -174,9 +174,6 @@ public void join() throws InterruptedException { public void stop() throws Exception { httpServer.stop(); - if (prometheusMetricsSink != null) { - DefaultMetricsSystem.instance().unregisterSource("prometheus"); - } } public URL getUrl() {