Index status / Report unreadable cluster health responses

index/src/main/java/org/fao/geonet/index/es/EsRestClient.java

@@ -41,6 +41,8 @@
 import co.elastic.clients.json.JsonpMapper;
 import co.elastic.clients.json.jackson.JacksonJsonpMapper;
 import co.elastic.clients.transport.ElasticsearchTransport;
+import co.elastic.clients.transport.TransportException;
+import co.elastic.clients.transport.Version;
 import co.elastic.clients.transport.rest_client.RestClientTransport;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -60,6 +62,8 @@
 import org.apache.http.nio.conn.SchemeIOSessionStrategy;
 import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;
 import org.apache.http.ssl.SSLContextBuilder;
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
 import org.fao.geonet.utils.Log;
@@ -86,6 +90,10 @@ public class EsRestClient implements InitializingBean {
 
     private ElasticsearchAsyncClient asyncClient;
 
+    private RestClient restClient;
+
+    private boolean healthDecodeFailureReported = false;
+
 
     private String serverUrl;
 
@@ -176,7 +184,7 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate
                 }
             }
 
-            RestClient restClient = builder.build();
+            restClient = builder.build();
 
             ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
 
@@ -527,9 +535,53 @@ public static String analyzeField(String collection,
 
     // TODO: check index exist too
     public String getServerStatus() throws IOException {
+        try {
+            HealthResponse response = client.cluster().health();
+            return response.status().toString();
+        } catch (TransportException e) {
+            // The typed client only decodes the health response of the server version it is built for.
+            // Any other version may return a response with missing or unknown properties, so read the
+            // status with the low level client which does not check the response against a model.
+            logHealthDecodeFailure(e);
+            try {
+                return getServerStatusUsingLowLevelClient();
+            } catch (Exception fallbackException) {
+                e.addSuppressed(fallbackException);
+                throw e;
+            }
+        }
+    }
 
-        HealthResponse response = client.cluster().health();
-        return response.status().toString();
+    /**
+     * Read the cluster status from the raw <code>_cluster/health</code> response.
+     */
+    private String getServerStatusUsingLowLevelClient() throws IOException {
+        Response response = restClient.performRequest(new Request("GET", "/_cluster/health"));
+        JsonNode status = new ObjectMapper().readTree(response.getEntity().getContent()).get("status");
+        if (status == null) {
+            throw new IOException(String.format(
+                "No status property found in the cluster health response from %s.", serverUrl));
+        }
+        return status.asText();
+    }
+
+    /**
+     * The status is checked on a regular basis, so only report the decoding error once.
+     * It is reported as an error because the default log configuration only reports
+     * errors for the index.
+     */
+    private void logHealthDecodeFailure(TransportException e) {
+        String message = String.format(
+            "Failed to decode the cluster health response returned by %s using the Elasticsearch client %s. "
+                + "Check that the index server version is compatible with this GeoNetwork version. "
+                + "Reading the cluster status using the low level client. Error is %s.",
+            serverUrl, Version.VERSION, e.getMessage());
+        if (healthDecodeFailureReported) {
+            Log.debug("geonetwork.index", message);
+        } else {
+            healthDecodeFailureReported = true;
+            Log.error("geonetwork.index", message);
+        }
     }
 
     public String getServerVersion() throws IOException, ElasticsearchException {

index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2001-2016 Food and Agriculture Organization of the
+ * Copyright (C) 2001-2026 Food and Agriculture Organization of the
  * United Nations (FAO-UN), United Nations World Food Programme (WFP)
  * and United Nations Environment Programme (UNEP)
  *
@@ -22,9 +22,12 @@
  */
 package org.fao.geonet.index.es;
 
+import co.elastic.clients.transport.TransportException;
+import co.elastic.clients.transport.Version;
 import org.fao.geonet.index.IServerStatusChecker;
 import org.fao.geonet.index.State;
 import org.fao.geonet.index.Status;
+import org.fao.geonet.utils.Log;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -34,11 +37,15 @@ public class EsServerStatusChecker
     extends QuartzJobBean
     implements IServerStatusChecker {
 
+    private static final String LOGGER = "geonetwork.index";
+
     @Autowired
     private Status status;
 
     private boolean indexChecked = false;
 
+    private boolean versionChecked = false;
+
     public EsServerStatusChecker() {
     }
 
@@ -60,6 +67,7 @@ public Status checkState() {
         String status = null;
         try {
             status = client.getServerStatus();
+            checkServerVersion();
             if ("green".equalsIgnoreCase(status)) {
                 this.status.setState(State.GREEN, "Index up and running. All green.");
                 checkIndexState();
@@ -69,13 +77,61 @@ public Status checkState() {
             } else {
                 this.status.setState(State.RED, "Index is down");
             }
+        } catch (TransportException e) {
+            this.status.setState(State.RED, String.format(
+                "Unable to read the status of the index at %s. The response can not be decoded by the Elasticsearch "
+                    + "client %s. Check that the index server version is compatible with this GeoNetwork version. "
+                    + "Error is %s", client.getServerUrl(), Version.VERSION, e.getMessage()));
         } catch (Exception e) {
             this.status.setState(State.UNINITIALIZED, String.format(
                 "Unable to revive connection to %s. Error is %s", client.getServerUrl(), e.getMessage()));
         }
         return this.status;
     }
 
+    /**
+     * Report once the version of the index server when it is not the version of the client
+     * GeoNetwork is built with. The check is only made when the server can be reached, and
+     * is retried on the next run when the version can not be read.
+     */
+    private void checkServerVersion() {
+        if (versionChecked) {
+            return;
+        }
+        String serverVersion;
+        try {
+            serverVersion = client.getServerVersion();
+        } catch (Exception e) {
+            Log.debug(LOGGER, String.format(
+                "Unable to read the version of the index server at %s. Error is %s",
+                client.getServerUrl(), e.getMessage()));
+            return;
+        }
+        versionChecked = true;
+
+        String message = versionMismatchMessage(client.getServerUrl(), serverVersion);
+        if (message != null) {
+            Log.error(LOGGER, message);
+        }
+    }
+
+    /**
+     * @return the message to report when the index server is not a version supported
+     * by the Elasticsearch client this GeoNetwork is built with, null when it is
+     * supported or when the version can not be parsed.
+     */
+    static String versionMismatchMessage(String serverUrl, String serverVersion) {
+        Version server = Version.parse(serverVersion);
+        if (server == null || server.major() == Version.VERSION.major()) {
+            return null;
+        }
+        return String.format(
+            "Index server at %s is Elasticsearch %s but this GeoNetwork version is built with the Elasticsearch "
+                + "client %s. Only Elasticsearch %d.x is supported, check the installation guide. "
+                + "Running another version leads to errors which are not always reported as a version issue.",
+            serverUrl, serverVersion, Version.VERSION, Version.VERSION.major());
+    }
+
     @Override
     public Status checkIndexState() {
         if (!indexChecked) {