HDDS-16019. Publish httpfs gateway metrics to Prometheus via /prom endpoint

hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/BaseHttpServer.java

@@ -155,26 +155,7 @@ public BaseHttpServer(MutableConfigurationSource conf, String name)
                                     conf.getBoolean(HddsConfigKeys.HDDS_PROFILER_ENABLED, false);
 
       if (prometheusSupport) {
-        prometheusMetricsSink = new PrometheusMetricsSink(name);
-        httpServer.getWebAppContext().getServletContext()
-            .setAttribute(PROMETHEUS_SINK, prometheusMetricsSink);
-        HddsPrometheusConfig prometheusConfig =
-            conf.getObject(HddsPrometheusConfig.class);
-        String token = prometheusConfig.getPrometheusEndpointToken();
-        if (StringUtils.isNotEmpty(token)) {
-          httpServer.getWebAppContext().getServletContext()
-              .setAttribute(PrometheusServlet.SECURITY_TOKEN, token);
-          // Adding as internal servlet since we want to have token based
-          // auth and hence SPNEGO should be disabled if security is enabled.
-          httpServer.addInternalServlet("prometheus", "/prom",
-              PrometheusServlet.class);
-        } else {
-          // If token is not configured, keeping as regular servlet and not
-          // internal servlet since we do not want to expose /prom endpoint
-          // without authentication in a secure cluster.
-          httpServer.addServlet("prometheus", "/prom",
-              PrometheusServlet.class);
-        }
+        prometheusMetricsSink = addPrometheusEndpoint(httpServer, conf, name);
       }
 
       if (profilerSupport) {
@@ -243,6 +224,27 @@ public static HttpServer2.Builder newHttpServer2BuilderForOzone(
     return builder;
   }
 
+  /**
+   * Add the Prometheus endpoint at {@code /prom} to an existing {@link HttpServer2} instance.
+   * Returns the registered sink for use in start/stop lifecycle calls.
+   */
+  public static PrometheusMetricsSink addPrometheusEndpoint(
+      HttpServer2 httpServer, ConfigurationSource conf, String name) {
+    PrometheusMetricsSink sink = new PrometheusMetricsSink(name);
+    httpServer.getWebAppContext().getServletContext().setAttribute(PROMETHEUS_SINK, sink);
+    String token = conf.getObject(HddsPrometheusConfig.class).getPrometheusEndpointToken();
+    if (StringUtils.isNotEmpty(token)) {
+      httpServer.getWebAppContext().getServletContext()
+          .setAttribute(PrometheusServlet.SECURITY_TOKEN, token);
+      // Token-based auth: use internal servlet so SPNEGO is bypassed.
+      httpServer.addInternalServlet("prometheus", "/prom", PrometheusServlet.class);
+    } else {
+      // No token: regular servlet, protected by the server's auth filter in secure mode.
+      httpServer.addServlet("prometheus", "/prom", PrometheusServlet.class);
+    }
+    return sink;
+  }
+
   /**
    * Add a servlet to BaseHttpServer.
    *

hadoop-ozone/dist/src/main/compose/ozone/monitoring.yaml

@@ -52,3 +52,5 @@ services:
     <<: *monitoring-config
   recon:
     <<: *monitoring-config
+  httpfs:
+    <<: *monitoring-config

hadoop-ozone/dist/src/main/compose/ozone/prometheus.yml

@@ -61,3 +61,12 @@ scrape_configs:
          - "recon:9888"
        labels:
          component: recon
+  - job_name: ozone-httpfs
+    metrics_path: /prom
+    params:
+      user.name: [hadoop]
+    static_configs:
+      - targets:
+          - "httpfs:14000"
+        labels:
+          component: httpfs

hadoop-ozone/httpfsgateway/src/main/java/org/apache/ozone/fs/http/server/HttpFSServerWebServer.java

@@ -28,10 +28,14 @@
 import java.util.LinkedHashSet;
 import java.util.Set;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdds.HddsConfigKeys;
 import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.http.BaseHttpServer;
 import org.apache.hadoop.hdds.server.http.HttpServer2;
+import org.apache.hadoop.hdds.server.http.PrometheusMetricsSink;
 import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.security.AuthenticationFilterInitializer;
 import org.apache.hadoop.security.authentication.server.ProxyUserAuthenticationFilterInitializer;
 import org.apache.hadoop.security.authorize.AccessControlList;
@@ -72,6 +76,7 @@ public class HttpFSServerWebServer {
 
   private final HttpServer2 httpServer;
   private final String scheme;
+  private PrometheusMetricsSink prometheusMetricsSink;
 
   HttpFSServerWebServer(OzoneConfiguration conf, Configuration sslConf) throws
       Exception {
@@ -129,6 +134,10 @@ public class HttpFSServerWebServer {
         .setACL(new AccessControlList(conf.get(HTTP_ADMINS_KEY, " ")))
         .addEndpoint(endpoint)
         .build();
+
+    if (conf.getBoolean(HddsConfigKeys.HDDS_PROMETHEUS_ENABLED, true)) {
+      prometheusMetricsSink = BaseHttpServer.addPrometheusEndpoint(httpServer, conf, NAME);
+    }
   }
 
   /**
@@ -153,6 +162,10 @@ private static void deprecateEnv(String varName, Configuration conf,
 
   public void start() throws IOException {
     httpServer.start();
+    if (prometheusMetricsSink != null) {
+      DefaultMetricsSystem.instance()
+          .register("prometheus", "Hadoop metrics prometheus exporter", prometheusMetricsSink);
+    }
   }
 
   public void join() throws InterruptedException {