fix(metrics): fix GFE and AFE metrics publishing (#17561)

## Description
This PR enables and properly extracts Spanner Google Front End (GFE) and
Application Front End (AFE) latency metrics.
### Key Changes:
- **Fix Metadata Extraction Logic**: Modified MetricsInterceptor and
AsyncMetricsInterceptor to strictly extract server-timing data from
initial_metadata().
- **Add AFE Metrics Publishing**: Extended MetricsTracer to parse AFE
latency (afe;\s*dur=) from the server-timing header alongside GFE
metrics. Two new OpenTelemetry instruments (afe_latency and
afe_missing_header_count) have been introduced to publish these.
- **Support for Streaming RPCs**: Removed the manual toggle in
SpannerMetricsTracerFactory. GFE and AFE metrics capture is now
always-on whenever OpenTelemetry tracing is enabled.
- **Enable Frontend Metrics by Default**: Removed the `gfe_enabled`
toggle in `SpannerMetricsTracerFactory`. GFE metrics capture is now
always-on whenever OpenTelemetry tracing is enabled.
- **Testing**: Added unit tests and a new mockserver test
(test_frontend_metrics.py) to ensure both GFE and AFE metrics are
correctly published end-to-end.
- Local Testing Screenshot:
https://screenshot.googleplex.com/image/4tmjwuPjzdttcjW.png

Author: sinhasubham

packages/google-cloud-spanner/google/cloud/spanner_v1/_helpers.py

@@ -20,6 +20,7 @@
 import logging
 import math
 import operator
+import os
 import threading
 import time
 import uuid
@@ -69,6 +70,11 @@
 import random
 from typing import List, Tuple
 
+ENABLE_AFE_SERVER_TIMING = (
+    os.environ.get("SPANNER_DISABLE_AFE_SERVER_TIMING", "").lower() != "true"
+    and os.environ.get("SPANNER_DISABLE_BUILTIN_METRICS", "").lower() != "true"
+)
+
 # Validation error messages
 NUMERIC_MAX_SCALE_ERR_MSG = (
     "Max scale for a numeric is 9. The requested numeric has scale {}"
@@ -707,6 +713,13 @@ def __init__(self, session):
         self._session = session
 
 
+def _append_routing_headers(metadata):
+    """Appends routing and backend-specific headers to the metadata."""
+    if ENABLE_AFE_SERVER_TIMING:
+        metadata.append(("x-goog-spanner-enable-afe-server-timing", "true"))
+    return metadata
+
+
 def _metadata_with_prefix(prefix, **kw):
     """Create RPC metadata containing a prefix.
 
@@ -716,7 +729,8 @@ def _metadata_with_prefix(prefix, **kw):
     Returns:
         List[Tuple[str, str]]: RPC metadata with supplied prefix
     """
-    return [("google-cloud-resource-prefix", prefix)]
+    metadata = [("google-cloud-resource-prefix", prefix)]
+    return _append_routing_headers(metadata)
 
 
 def _retry_on_aborted_exception(

packages/google-cloud-spanner/google/cloud/spanner_v1/metrics/README.md

@@ -1,4 +1,4 @@
-# Custom Metric Exporter
+# Custom Metric Exporter
 The custom metric exporter, as defined in [metrics_exporter.py](./metrics_exporter.py), is designed to work in conjunction with OpenTelemetry and the Spanner client. It converts data into its protobuf equivalent and sends it to Google Cloud Monitoring.
 
 ## Filtering Criteria
@@ -10,8 +10,10 @@ The exporter filters metrics based on the following conditions, utilizing values
   * `attempt_count`
   * `operation_latencies`
   * `operation_count`
-  * `gfe_latency`
-  * `gfe_missing_header_count`
+  * `gfe_latencies`
+  * `gfe_connectivity_error_count`
+  * `afe_latencies`
+  * `afe_connectivity_error_count`
 
 ## Service Endpoint
 The exporter sends metrics to the Google Cloud Monitoring [service endpoint](https://cloud.google.com/python/docs/reference/monitoring/latest/google.cloud.monitoring_v3.services.metric_service.MetricServiceClient#google_cloud_monitoring_v3_services_metric_service_MetricServiceClient_create_service_time_series), distinct from the regular client endpoint. This service endpoint operates under a different quota limit than the user endpoint and features an additional server-side filter that only permits a predefined set of metrics to pass through.

packages/google-cloud-spanner/google/cloud/spanner_v1/metrics/constants.py

@@ -58,13 +58,19 @@
 METRIC_NAME_ATTEMPT_LATENCIES = "attempt_latencies"
 METRIC_NAME_OPERATION_COUNT = "operation_count"
 METRIC_NAME_ATTEMPT_COUNT = "attempt_count"
-METRIC_NAME_GFE_LATENCY = "gfe_latency"
-METRIC_NAME_GFE_MISSING_HEADER_COUNT = "gfe_missing_header_count"
+METRIC_NAME_GFE_LATENCY = "gfe_latencies"
+METRIC_NAME_GFE_CONNECTIVITY_ERROR_COUNT = "gfe_connectivity_error_count"
+METRIC_NAME_AFE_LATENCY = "afe_latencies"
+METRIC_NAME_AFE_CONNECTIVITY_ERROR_COUNT = "afe_connectivity_error_count"
 METRIC_NAMES = [
     METRIC_NAME_OPERATION_LATENCIES,
     METRIC_NAME_ATTEMPT_LATENCIES,
     METRIC_NAME_OPERATION_COUNT,
     METRIC_NAME_ATTEMPT_COUNT,
+    METRIC_NAME_GFE_LATENCY,
+    METRIC_NAME_GFE_CONNECTIVITY_ERROR_COUNT,
+    METRIC_NAME_AFE_LATENCY,
+    METRIC_NAME_AFE_CONNECTIVITY_ERROR_COUNT,
 ]
 
 METRIC_EXPORT_INTERVAL_MS = 60000  # 1 Minute