From 24a48d76ed211b5dce4a78a939731f43f35bc308 Mon Sep 17 00:00:00 2001 From: Juan Luis Rodriguez Ponce Date: Fri, 14 Aug 2026 12:12:10 +0200 Subject: [PATCH 1/2] Index status / Report unreadable cluster health responses EsServerStatusChecker reads the cluster health through the typed Elasticsearch client. That client only decodes the health response of the server version it is built for: 8.19 requires unassigned_primary_shards, a property added to _cluster/health in Elasticsearch 8.16. Against an older server the call fails with a transport error although the request returned a 200. The catch all then sets the state to uninitialized and the catalogue reports "Unable to revive connection to http://host:9200", which points at connectivity while the server is in fact reachable and search keeps working through the search proxy. Read the status with the low level client when the typed client can not decode the response, so the reported state matches the state of the cluster. When that fails too, report the decoding failure with a message pointing at the version of the index server instead of the connection. The decoding failure is logged once, the check runs every five seconds by default. --- .../org/fao/geonet/index/es/EsRestClient.java | 58 ++++++- .../index/es/EsServerStatusChecker.java | 9 +- .../index/es/EsServerStatusCheckerTest.java | 148 ++++++++++++++++++ 3 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java diff --git a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java index 715431687031..3b8fe89643e4 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsRestClient.java @@ -41,6 +41,8 @@ import co.elastic.clients.json.JsonpMapper; import co.elastic.clients.json.jackson.JacksonJsonpMapper; import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.TransportException; +import co.elastic.clients.transport.Version; import co.elastic.clients.transport.rest_client.RestClientTransport; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -60,6 +62,8 @@ import org.apache.http.nio.conn.SchemeIOSessionStrategy; import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContextBuilder; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.fao.geonet.utils.Log; @@ -86,6 +90,10 @@ public class EsRestClient implements InitializingBean { private ElasticsearchAsyncClient asyncClient; + private RestClient restClient; + + private boolean healthDecodeFailureReported = false; + private String serverUrl; @@ -176,7 +184,7 @@ public boolean isTrusted(X509Certificate[] arg0, String arg1) throws Certificate } } - RestClient restClient = builder.build(); + restClient = builder.build(); ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); @@ -527,9 +535,53 @@ public static String analyzeField(String collection, // TODO: check index exist too public String getServerStatus() throws IOException { + try { + HealthResponse response = client.cluster().health(); + return response.status().toString(); + } catch (TransportException e) { + // The typed client only decodes the health response of the server version it is built for. + // Any other version may return a response with missing or unknown properties, so read the + // status with the low level client which does not check the response against a model. + logHealthDecodeFailure(e); + try { + return getServerStatusUsingLowLevelClient(); + } catch (Exception fallbackException) { + e.addSuppressed(fallbackException); + throw e; + } + } + } - HealthResponse response = client.cluster().health(); - return response.status().toString(); + /** + * Read the cluster status from the raw _cluster/health response. + */ + private String getServerStatusUsingLowLevelClient() throws IOException { + Response response = restClient.performRequest(new Request("GET", "/_cluster/health")); + JsonNode status = new ObjectMapper().readTree(response.getEntity().getContent()).get("status"); + if (status == null) { + throw new IOException(String.format( + "No status property found in the cluster health response from %s.", serverUrl)); + } + return status.asText(); + } + + /** + * The status is checked on a regular basis, so only report the decoding error once. + * It is reported as an error because the default log configuration only reports + * errors for the index. + */ + private void logHealthDecodeFailure(TransportException e) { + String message = String.format( + "Failed to decode the cluster health response returned by %s using the Elasticsearch client %s. " + + "Check that the index server version is compatible with this GeoNetwork version. " + + "Reading the cluster status using the low level client. Error is %s.", + serverUrl, Version.VERSION, e.getMessage()); + if (healthDecodeFailureReported) { + Log.debug("geonetwork.index", message); + } else { + healthDecodeFailureReported = true; + Log.error("geonetwork.index", message); + } } public String getServerVersion() throws IOException, ElasticsearchException { diff --git a/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java b/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java index 50a4fd554f50..2b669e0bed2b 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2001-2016 Food and Agriculture Organization of the + * Copyright (C) 2001-2026 Food and Agriculture Organization of the * United Nations (FAO-UN), United Nations World Food Programme (WFP) * and United Nations Environment Programme (UNEP) * @@ -22,6 +22,8 @@ */ package org.fao.geonet.index.es; +import co.elastic.clients.transport.TransportException; +import co.elastic.clients.transport.Version; import org.fao.geonet.index.IServerStatusChecker; import org.fao.geonet.index.State; import org.fao.geonet.index.Status; @@ -69,6 +71,11 @@ public Status checkState() { } else { this.status.setState(State.RED, "Index is down"); } + } catch (TransportException e) { + this.status.setState(State.RED, String.format( + "Unable to read the status of the index at %s. The response can not be decoded by the Elasticsearch " + + "client %s. Check that the index server version is compatible with this GeoNetwork version. " + + "Error is %s", client.getServerUrl(), Version.VERSION, e.getMessage())); } catch (Exception e) { this.status.setState(State.UNINITIALIZED, String.format( "Unable to revive connection to %s. Error is %s", client.getServerUrl(), e.getMessage())); diff --git a/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java b/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java new file mode 100644 index 000000000000..2f3f566fa79a --- /dev/null +++ b/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (C) 2001-2026 Food and Agriculture Organization of the + * United Nations (FAO-UN), United Nations World Food Programme (WFP) + * and United Nations Environment Programme (UNEP) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or (at + * your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + * + * Contact: Jeroen Ticheler - FAO - Viale delle Terme di Caracalla 2, + * Rome - Italy. email: geonetwork@osgeo.org + */ +package org.fao.geonet.index.es; + +import co.elastic.clients.transport.TransportException; +import co.elastic.clients.transport.http.TransportHttpClient; +import co.elastic.clients.util.BinaryData; +import org.fao.geonet.index.State; +import org.fao.geonet.index.Status; +import org.junit.Test; + +import java.io.IOException; +import java.net.ConnectException; +import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class EsServerStatusCheckerTest { + + private static final String SERVER_URL = "http://localhost:9200"; + + @Test + public void serverStatusIsReported() { + Status status = check(new EsRestClientStub("green", null)); + assertEquals(State.GREEN, status.getState()); + + status = check(new EsRestClientStub("yellow", null)); + assertEquals(State.YELLOW, status.getState()); + + status = check(new EsRestClientStub("red", null)); + assertEquals(State.RED, status.getState()); + } + + @Test + public void connectionFailureIsReportedAsUninitialized() { + Status status = check(new EsRestClientStub(null, new ConnectException("Connection refused"))); + + assertEquals(State.UNINITIALIZED, status.getState()); + assertTrue(status.getMessage(), status.getMessage().startsWith("Unable to revive connection to " + SERVER_URL)); + } + + /** + * A response which the client can not decode is not a connectivity problem, it usually means + * that the server version is not the one the client is built for. + */ + @Test + public void decodingFailureIsReportedAsVersionIssue() { + Status status = check(new EsRestClientStub(null, transportException())); + + assertEquals(State.RED, status.getState()); + assertTrue(status.getMessage(), status.getMessage().contains("can not be decoded")); + assertTrue(status.getMessage(), status.getMessage().contains("index server version is compatible")); + } + + private Status check(EsRestClient client) { + EsServerStatusChecker checker = new EsServerStatusChecker(); + checker.setStatus(new Status("index")); + checker.client = client; + return checker.checkState(); + } + + private TransportException transportException() { + return new TransportException(new ResponseStub(), "Failed to decode response", "es/cluster.health"); + } + + /** + * Returns the configured status or throws the configured error. + */ + private static class EsRestClientStub extends EsRestClient { + private final String serverStatus; + private final IOException error; + + EsRestClientStub(String serverStatus, IOException error) { + this.serverStatus = serverStatus; + this.error = error; + setServerUrl(SERVER_URL); + } + + @Override + public String getServerStatus() throws IOException { + if (error != null) { + throw error; + } + return serverStatus; + } + } + + /** + * Minimal response required to build a {@link TransportException}. + */ + private static class ResponseStub implements TransportHttpClient.Response { + @Override + public TransportHttpClient.Node node() { + return new TransportHttpClient.Node(SERVER_URL); + } + + @Override + public int statusCode() { + return 200; + } + + @Override + public String header(String name) { + return null; + } + + @Override + public List headers(String name) { + return Collections.emptyList(); + } + + @Override + public BinaryData body() { + return null; + } + + @Override + public Object originalResponse() { + return null; + } + + @Override + public void close() { + } + } +} From c5b6812021c8e57a69618285677b4560b39cf2c2 Mon Sep 17 00:00:00 2001 From: Juan Luis Rodriguez Ponce Date: Fri, 14 Aug 2026 12:46:36 +0200 Subject: [PATCH 2/2] Index status / Report the version of the index server The version of the index server is only visible in the site information page. When it is not the version the Elasticsearch client is built for, the failures show up somewhere else entirely, as a response which can not be decoded or as a query the server rejects, and nothing points at the version of the server. Compare the major version of the server with the version of the client once the server can be reached, and report a difference once. The version is read again on the next run when the server can not be reached, so a catalogue started before its index server still reports it. --- .../index/es/EsServerStatusChecker.java | 49 +++++++++++++ .../index/es/EsServerStatusCheckerTest.java | 70 +++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java b/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java index 2b669e0bed2b..a6b02c6fc88d 100644 --- a/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java +++ b/index/src/main/java/org/fao/geonet/index/es/EsServerStatusChecker.java @@ -27,6 +27,7 @@ import org.fao.geonet.index.IServerStatusChecker; import org.fao.geonet.index.State; import org.fao.geonet.index.Status; +import org.fao.geonet.utils.Log; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; @@ -36,11 +37,15 @@ public class EsServerStatusChecker extends QuartzJobBean implements IServerStatusChecker { + private static final String LOGGER = "geonetwork.index"; + @Autowired private Status status; private boolean indexChecked = false; + private boolean versionChecked = false; + public EsServerStatusChecker() { } @@ -62,6 +67,7 @@ public Status checkState() { String status = null; try { status = client.getServerStatus(); + checkServerVersion(); if ("green".equalsIgnoreCase(status)) { this.status.setState(State.GREEN, "Index up and running. All green."); checkIndexState(); @@ -83,6 +89,49 @@ public Status checkState() { return this.status; } + /** + * Report once the version of the index server when it is not the version of the client + * GeoNetwork is built with. The check is only made when the server can be reached, and + * is retried on the next run when the version can not be read. + */ + private void checkServerVersion() { + if (versionChecked) { + return; + } + String serverVersion; + try { + serverVersion = client.getServerVersion(); + } catch (Exception e) { + Log.debug(LOGGER, String.format( + "Unable to read the version of the index server at %s. Error is %s", + client.getServerUrl(), e.getMessage())); + return; + } + versionChecked = true; + + String message = versionMismatchMessage(client.getServerUrl(), serverVersion); + if (message != null) { + Log.error(LOGGER, message); + } + } + + /** + * @return the message to report when the index server is not a version supported + * by the Elasticsearch client this GeoNetwork is built with, null when it is + * supported or when the version can not be parsed. + */ + static String versionMismatchMessage(String serverUrl, String serverVersion) { + Version server = Version.parse(serverVersion); + if (server == null || server.major() == Version.VERSION.major()) { + return null; + } + return String.format( + "Index server at %s is Elasticsearch %s but this GeoNetwork version is built with the Elasticsearch " + + "client %s. Only Elasticsearch %d.x is supported, check the installation guide. " + + "Running another version leads to errors which are not always reported as a version issue.", + serverUrl, serverVersion, Version.VERSION, Version.VERSION.major()); + } + @Override public Status checkIndexState() { if (!indexChecked) { diff --git a/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java b/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java index 2f3f566fa79a..4d9167aa4648 100644 --- a/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java +++ b/index/src/test/java/org/fao/geonet/index/es/EsServerStatusCheckerTest.java @@ -23,6 +23,7 @@ package org.fao.geonet.index.es; import co.elastic.clients.transport.TransportException; +import co.elastic.clients.transport.Version; import co.elastic.clients.transport.http.TransportHttpClient; import co.elastic.clients.util.BinaryData; import org.fao.geonet.index.State; @@ -35,6 +36,8 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class EsServerStatusCheckerTest { @@ -74,6 +77,61 @@ public void decodingFailureIsReportedAsVersionIssue() { assertTrue(status.getMessage(), status.getMessage().contains("index server version is compatible")); } + @Test + public void versionOfTheServerIsReportedWhenItIsNotTheVersionOfTheClient() { + String message = EsServerStatusChecker.versionMismatchMessage(SERVER_URL, "7.17.15"); + + assertNotNull(message); + assertTrue(message, message.contains("7.17.15")); + assertTrue(message, message.contains(Version.VERSION.toString())); + } + + @Test + public void versionOfTheServerIsNotReportedWhenItIsSupported() { + assertNull(EsServerStatusChecker.versionMismatchMessage(SERVER_URL, Version.VERSION.toString())); + assertNull(EsServerStatusChecker.versionMismatchMessage(SERVER_URL, + Version.VERSION.major() + ".0.0")); + } + + @Test + public void versionOfTheServerIsNotReportedWhenItCanNotBeRead() { + assertNull(EsServerStatusChecker.versionMismatchMessage(SERVER_URL, "unknown")); + } + + /** + * The check runs every few seconds, the version of the server is only read once. + */ + @Test + public void versionOfTheServerIsOnlyReadOnce() { + EsRestClientStub client = new EsRestClientStub("green", null); + EsServerStatusChecker checker = new EsServerStatusChecker(); + checker.setStatus(new Status("index")); + checker.client = client; + + checker.checkState(); + checker.checkState(); + checker.checkState(); + + assertEquals(1, client.serverVersionReads); + } + + /** + * A server which can not be reached is checked again on the next run. + */ + @Test + public void versionOfTheServerIsReadAgainWhenItCanNotBeRead() { + EsRestClientStub client = new EsRestClientStub("green", null); + client.serverVersionError = new IOException("Connection refused"); + EsServerStatusChecker checker = new EsServerStatusChecker(); + checker.setStatus(new Status("index")); + checker.client = client; + + checker.checkState(); + checker.checkState(); + + assertEquals(2, client.serverVersionReads); + } + private Status check(EsRestClient client) { EsServerStatusChecker checker = new EsServerStatusChecker(); checker.setStatus(new Status("index")); @@ -92,6 +150,9 @@ private static class EsRestClientStub extends EsRestClient { private final String serverStatus; private final IOException error; + private IOException serverVersionError; + private int serverVersionReads = 0; + EsRestClientStub(String serverStatus, IOException error) { this.serverStatus = serverStatus; this.error = error; @@ -105,6 +166,15 @@ public String getServerStatus() throws IOException { } return serverStatus; } + + @Override + public String getServerVersion() throws IOException { + serverVersionReads++; + if (serverVersionError != null) { + throw serverVersionError; + } + return Version.VERSION.toString(); + } } /**