diff --git a/hadoop-hdds/docs/content/tools/_index.md b/hadoop-hdds/docs/content/tools/_index.md index 3a8bf08e826f..8bb437c8421f 100644 --- a/hadoop-hdds/docs/content/tools/_index.md +++ b/hadoop-hdds/docs/content/tools/_index.md @@ -43,8 +43,8 @@ Client commands: * **sh** - Primary command line interface for ozone to manage volumes/buckets/keys. * **fs** - Runs a command on ozone file system (similar to `hdfs dfs`) - * **local** - Runs a single-node local Ozone cluster (SCM, OM, and datanodes) - in one process for development. + * **local** - Runs a single-node local Ozone cluster (SCM, OM, datanodes, + and optional S3 Gateway and Recon) in one process for development. * **version** - Prints the version of Ozone and HDDS. diff --git a/hadoop-ozone/integration-test-recon/pom.xml b/hadoop-ozone/integration-test-recon/pom.xml index ca2fae43bf03..e01c81375a40 100644 --- a/hadoop-ozone/integration-test-recon/pom.xml +++ b/hadoop-ozone/integration-test-recon/pom.xml @@ -180,6 +180,11 @@ ozone-reconcodegen test + + org.apache.ozone + ozone-tools + test + org.apache.ratis ratis-common diff --git a/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneRecon.java b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneRecon.java new file mode 100644 index 000000000000..b6eb7f9dfd8a --- /dev/null +++ b/hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneRecon.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Integration tests for Recon in the {@code ozone local} runtime. + */ +class TestLocalOzoneRecon { + + @TempDir + private Path tempDir; + + @Test + void reconServesRequestsWhenEnabled() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + tempDir.resolve("local-ozone-recon")) + .setS3gEnabled(false) + .setReconEnabled(true) + .setStartupTimeout(Duration.ofMinutes(2)) + .build(); + + try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, new OzoneConfiguration())) { + cluster.start(); + + assertTrue(cluster.getReconPort() > 0); + assertHttpEndpointResponds(cluster.getReconEndpoint()); + } + } + + private static void assertHttpEndpointResponds(String endpoint) throws Exception { + HttpURLConnection connection = (HttpURLConnection) new URL(endpoint).openConnection(); + try { + connection.setConnectTimeout(1_000); + connection.setReadTimeout(1_000); + // Any HTTP response proves Recon is serving requests. + assertTrue(connection.getResponseCode() > 0, endpoint); + } finally { + connection.disconnect(); + } + } +} diff --git a/hadoop-ozone/integration-test-s3/pom.xml b/hadoop-ozone/integration-test-s3/pom.xml index 75c3e371c08d..fcceb04cf343 100644 --- a/hadoop-ozone/integration-test-s3/pom.xml +++ b/hadoop-ozone/integration-test-s3/pom.xml @@ -154,6 +154,11 @@ ozone-s3gateway test + + org.apache.ozone + ozone-tools + test + org.apache.ratis ratis-common diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java new file mode 100644 index 000000000000..e03ffd280cf1 --- /dev/null +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.local; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.file.Path; +import java.time.Duration; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Integration tests for the S3 Gateway of the {@code ozone local} runtime. + */ +class TestLocalOzoneS3 { + + @TempDir + private Path tempDir; + + @Test + void s3GatewayServesRequests() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + tempDir.resolve("local-ozone-s3")) + .setStartupTimeout(Duration.ofMinutes(2)) + .build(); + + try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, new OzoneConfiguration())) { + cluster.start(); + + assertTrue(cluster.getS3gPort() > 0); + assertS3EndpointResponds(cluster.getS3Endpoint()); + } + } + + private static void assertS3EndpointResponds(String endpoint) throws Exception { + HttpURLConnection connection = (HttpURLConnection) new URL(endpoint).openConnection(); + try { + connection.setConnectTimeout(1_000); + connection.setReadTimeout(1_000); + // Any HTTP response proves the S3 Gateway is serving requests. + assertTrue(connection.getResponseCode() > 0, endpoint); + } finally { + connection.disconnect(); + } + } +} diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java index c440ba25850e..19c2144c1798 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.recon; -import com.google.common.annotations.VisibleForTesting; import com.google.inject.Provider; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration.DeprecationDelta; @@ -50,16 +49,14 @@ private static void addDeprecations() { }); } - @VisibleForTesting public static void setConfiguration(OzoneConfiguration conf) { - // Nullity check is used in case the configuration was already set - // in the MiniOzoneCluster + // Nullity check is used in case the configuration was already set by a + // same-JVM launcher (MiniOzoneCluster or ozone local) if (configuration == null) { ConfigurationProvider.configuration = conf; } } - @VisibleForTesting public static void resetConfiguration() { configuration = null; } diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java index 88652f0457aa..9a201530fa50 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone.s3; -import com.google.common.annotations.VisibleForTesting; import javax.enterprise.inject.Produces; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -39,17 +38,15 @@ public static OzoneConfiguration configuration() { return configuration; } - @VisibleForTesting public static void setConfiguration( OzoneConfiguration conf) { - // Nullity check is used in case the configuration was already set - // in the MiniOzoneCluster + // Nullity check is used in case the configuration was already set by a + // same-JVM launcher (MiniOzoneCluster or ozone local) if (configuration == null) { OzoneConfigurationHolder.configuration = conf; } } - @VisibleForTesting public static void resetConfiguration() { configuration = null; } diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml index 3ffab37b2b24..432f320c2ecc 100644 --- a/hadoop-ozone/tools/pom.xml +++ b/hadoop-ozone/tools/pom.xml @@ -82,6 +82,14 @@ org.apache.ozone ozone-manager + + org.apache.ozone + ozone-recon + + + org.apache.ozone + ozone-s3gateway + org.apache.ratis ratis-common diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java index 89961e8243ad..8972b16bdde3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java @@ -25,6 +25,12 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_BIND_HOST_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY; @@ -72,11 +78,26 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_HTTPS_BIND_HOST_KEY; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_HTTP_BIND_HOST_KEY; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR; +import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_BIND_HOST_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_BIND_HOST_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTPS_BIND_HOST_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_BIND_HOST_KEY; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.HttpURLConnection; import java.net.ServerSocket; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; @@ -90,8 +111,14 @@ import java.util.Properties; import java.util.Set; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.function.BooleanSupplier; import java.util.stream.Stream; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; @@ -107,13 +134,18 @@ import org.apache.hadoop.ozone.container.replication.ReplicationServer; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.helpers.S3SecretValue; +import org.apache.hadoop.ozone.recon.ConfigurationProvider; +import org.apache.hadoop.ozone.recon.ReconServer; +import org.apache.hadoop.ozone.recon.ReconSqlDbConfig; +import org.apache.hadoop.ozone.s3.Gateway; +import org.apache.hadoop.ozone.s3.OzoneConfigurationHolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Starts the SCM, OM, and datanode portion of the {@code ozone local} runtime. - * - *

S3 Gateway and Recon are added by later local runtime tickets.

+ * Starts the SCM, OM, datanode, and optional S3 Gateway and Recon portion of the {@code ozone local} + * runtime. */ public final class LocalOzoneCluster implements LocalOzoneRuntime { @@ -141,6 +173,14 @@ public final class LocalOzoneCluster implements LocalOzoneRuntime { private static final String OM_HTTP_PORT_KEY = "om.http"; private static final String OM_HTTPS_PORT_KEY = "om.https"; private static final String OM_RATIS_PORT_KEY = "om.ratis"; + private static final String S3G_HTTP_PORT_KEY = "s3g.http"; + private static final String S3G_HTTPS_PORT_KEY = "s3g.https"; + private static final String S3G_WEBADMIN_HTTP_PORT_KEY = "s3g.web.http"; + private static final String S3G_WEBADMIN_HTTPS_PORT_KEY = "s3g.web.https"; + private static final String RECON_HTTP_PORT_KEY = "recon.http"; + private static final String RECON_HTTPS_PORT_KEY = "recon.https"; + private static final String RECON_DATANODE_PORT_KEY = "recon.datanode"; + private static final String RECON_DIR_NAME = "recon"; private static final String DATANODE_PORT_KEY_PREFIX = "dn."; private static final String DATANODE_HTTP_PORT_KEY_SUFFIX = "http"; private static final String DATANODE_CLIENT_PORT_KEY_SUFFIX = "client"; @@ -195,6 +235,8 @@ public final class LocalOzoneCluster implements LocalOzoneRuntime { private PreparedConfiguration preparedConfiguration; private StorageContainerManager scm; private OzoneManager om; + private Gateway s3Gateway; + private ReconServer reconServer; private final List datanodes = new ArrayList<>(); private boolean previousMetricsMiniClusterMode; private boolean metricsMiniClusterModeEnabled; @@ -226,6 +268,17 @@ public void start() throws Exception { startOm(prepared.getConfiguration()); startDatanodes(prepared.getDatanodeConfigurations()); waitForClusterReadiness(config.getStartupTimeout()); + if (config.isReconEnabled()) { + startRecon(prepared.getConfiguration()); + waitForHttpEndpointReadiness(getReconEndpoint(), "Recon", + config.getStartupTimeout()); + } + if (config.isS3gEnabled()) { + provisionS3Credentials(); + startS3Gateway(prepared.getConfiguration()); + waitForHttpEndpointReadiness(getS3Endpoint(), "S3 Gateway", + config.getStartupTimeout()); + } } catch (Exception ex) { // Roll back without latching closed: the caller's close() still owns the // ephemeral data dir lifecycle. @@ -248,12 +301,14 @@ PreparedConfiguration prepareConfiguration() throws IOException { PortAllocator portAllocator = new PortAllocator(); int scmPort = configureScm(conf, persistedPorts, portAllocator); int omPort = configureOm(conf, persistedPorts, portAllocator); + int s3gPort = configureS3Gateway(conf, persistedPorts, portAllocator); + int reconPort = configureRecon(conf, persistedPorts, portAllocator); List datanodeConfigurations = configureDatanodes(conf, persistedPorts, portAllocator); persistedPorts.store(); preparedConfiguration = new PreparedConfiguration(conf, scmPort, omPort, - datanodeConfigurations); + s3gPort, reconPort, datanodeConfigurations); return preparedConfiguration; } @@ -282,12 +337,39 @@ public int getDatanodeCount() { @Override public int getS3gPort() { - return -1; + if (!config.isS3gEnabled()) { + return -1; + } + if (s3Gateway != null) { + return s3Gateway.getHttpAddress().getPort(); + } + return preparedConfiguration == null ? config.getS3gPort() + : preparedConfiguration.getS3gPort(); } @Override public String getS3Endpoint() { - return ""; + if (!config.isS3gEnabled()) { + return ""; + } + return "http://" + getDisplayHost() + ":" + getS3gPort(); + } + + @Override + public int getReconPort() { + if (!config.isReconEnabled()) { + return -1; + } + return preparedConfiguration == null ? config.getReconPort() + : preparedConfiguration.getReconPort(); + } + + @Override + public String getReconEndpoint() { + if (!config.isReconEnabled()) { + return ""; + } + return "http://" + getDisplayHost() + ":" + getReconPort(); } @Override @@ -315,12 +397,33 @@ public void close() throws IOException { private void stopServices() { try { // Shutdown is best-effort so one failed service cannot leak the others. - IOUtils.closeQuietly(this::stopDatanodes, this::stopOm, this::stopScm); + IOUtils.closeQuietly(this::stopS3Gateway, this::stopRecon, this::stopDatanodes, this::stopOm, + this::stopScm); } finally { + // The holders are static, so clear them even if the services never started. + OzoneConfigurationHolder.resetConfiguration(); + ConfigurationProvider.resetConfiguration(); restoreSameJvmMetricsMode(); } } + private void stopS3Gateway() throws Exception { + Gateway service = s3Gateway; + s3Gateway = null; + if (service != null) { + service.stop(); + } + } + + private void stopRecon() { + ReconServer service = reconServer; + reconServer = null; + if (service != null) { + service.stop(); + service.join(); + } + } + private void stopDatanodes() { List stoppers = new ArrayList<>(); for (int i = datanodes.size() - 1; i >= 0; i--) { @@ -477,6 +580,80 @@ private void configureOmStorage(OzoneConfiguration conf) throws IOException { omMetadataDir.toString()); } + private int configureS3Gateway(OzoneConfiguration conf, + PersistedPortState persistedPorts, PortAllocator portAllocator) + throws IOException { + if (!config.isS3gEnabled()) { + return -1; + } + int s3gHttpPort = reservePort(portAllocator, persistedPorts, + S3G_HTTP_PORT_KEY, config.getS3gPort()); + int s3gHttpsPort = reservePort(portAllocator, persistedPorts, + S3G_HTTPS_PORT_KEY, 0); + int s3gWebHttpPort = reservePort(portAllocator, persistedPorts, + S3G_WEBADMIN_HTTP_PORT_KEY, 0); + int s3gWebHttpsPort = reservePort(portAllocator, persistedPorts, + S3G_WEBADMIN_HTTPS_PORT_KEY, 0); + + conf.set(OZONE_S3G_HTTP_ADDRESS_KEY, + address(config.getHost(), s3gHttpPort)); + conf.set(OZONE_S3G_HTTP_BIND_HOST_KEY, config.getBindHost()); + conf.set(OZONE_S3G_HTTPS_ADDRESS_KEY, + address(config.getHost(), s3gHttpsPort)); + conf.set(OZONE_S3G_HTTPS_BIND_HOST_KEY, config.getBindHost()); + conf.set(OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY, + address(config.getHost(), s3gWebHttpPort)); + conf.set(OZONE_S3G_WEBADMIN_HTTP_BIND_HOST_KEY, config.getBindHost()); + conf.set(OZONE_S3G_WEBADMIN_HTTPS_ADDRESS_KEY, + address(config.getHost(), s3gWebHttpsPort)); + conf.set(OZONE_S3G_WEBADMIN_HTTPS_BIND_HOST_KEY, config.getBindHost()); + return s3gHttpPort; + } + + private int configureRecon(OzoneConfiguration conf, + PersistedPortState persistedPorts, PortAllocator portAllocator) + throws IOException { + if (!config.isReconEnabled()) { + return -1; + } + Path reconDir = config.getDataDir().resolve(RECON_DIR_NAME); + Files.createDirectories(reconDir); + conf.setIfUnset(OZONE_RECON_DB_DIR, reconDir.toString()); + conf.setIfUnset(OZONE_RECON_OM_SNAPSHOT_DB_DIR, reconDir.toString()); + conf.setIfUnset(OZONE_RECON_SCM_DB_DIR, reconDir.toString()); + + ReconSqlDbConfig dbConfig = conf.getObject(ReconSqlDbConfig.class); + dbConfig.setJdbcUrl("jdbc:derby:" + + reconDir.resolve("ozone_recon_derby.db")); + conf.setFromObject(dbConfig); + + int reconHttpPort = reservePort(portAllocator, persistedPorts, + RECON_HTTP_PORT_KEY, config.getReconPort()); + int reconHttpsPort = reservePort(portAllocator, persistedPorts, + RECON_HTTPS_PORT_KEY, 0); + int reconDatanodePort = reservePort(portAllocator, persistedPorts, + RECON_DATANODE_PORT_KEY, 0); + + conf.set(OZONE_RECON_ADDRESS_KEY, + address(config.getHost(), reconDatanodePort)); + conf.set(OZONE_RECON_DATANODE_ADDRESS_KEY, + address(config.getHost(), reconDatanodePort)); + conf.set(OZONE_RECON_DATANODE_BIND_HOST_KEY, config.getBindHost()); + conf.set(OZONE_RECON_HTTP_ADDRESS_KEY, + address(config.getHost(), reconHttpPort)); + conf.set(OZONE_RECON_HTTP_BIND_HOST_KEY, config.getBindHost()); + conf.set(OZONE_RECON_HTTPS_ADDRESS_KEY, + address(config.getHost(), reconHttpsPort)); + conf.set(OZONE_RECON_HTTPS_BIND_HOST_KEY, config.getBindHost()); + // Recon's start-up blocks until it leaves safe mode, and an empty local + // cluster never leaves it on its own, so keep the fallback wait short. + // Use set(), not setIfUnset(): the generated ozone-recon-default.xml + // already supplies the 300s default, so setIfUnset() would do nothing here + // and Recon would block start-up for 5 minutes. + conf.set(OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD, "10s"); + return reconHttpPort; + } + private List configureDatanodes(OzoneConfiguration conf, PersistedPortState persistedPorts, PortAllocator portAllocator) throws IOException { @@ -642,15 +819,113 @@ private void startDatanodes(List datanodeConfigurations) { } } + /** + * Stores the recommended local credentials in OM so S3 clients can sign requests without a + * separate {@code ozone s3 getsecret} bootstrap step. + */ + private void provisionS3Credentials() throws IOException { + om.getS3SecretManager().storeSecret(config.getS3AccessKey(), + S3SecretValue.of(config.getS3AccessKey(), config.getS3SecretKey())); + } + + private void startS3Gateway(OzoneConfiguration conf) throws Exception { + // Gateway reads its configuration from the static holder, like MiniOzoneCluster does when + // running S3 Gateway in the same JVM. + OzoneConfigurationHolder.resetConfiguration(); + OzoneConfigurationHolder.setConfiguration(new OzoneConfiguration(conf)); + // Assign the field before execute() so a failed start can still be rolled + // back by stopServices(). + s3Gateway = new Gateway(); + int exitCode = executeWithinStartupTimeout("S3 Gateway", + () -> s3Gateway.execute(NO_ARGS)); + if (exitCode != 0) { + throw new IOException("Failed to start the local S3 Gateway. Exit code " + + exitCode + "."); + } + } + + /** + * Runs a blocking in-JVM {@code execute()} startup under the configured + * startup timeout. Recon and S3 Gateway start-up can block until the service + * leaves safe mode; an empty local cluster relies on a short safe-mode + * threshold to bound that, and this guarantees the launcher fails fast and + * rolls back instead of hanging if that block ever runs long. + */ + int executeWithinStartupTimeout(String serviceName, + Callable startup) throws Exception { + Duration timeout = config.getStartupTimeout(); + ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "local-startup-" + serviceName); + thread.setDaemon(true); + return thread; + }); + Future future = executor.submit(startup); + try { + return future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException ex) { + future.cancel(true); + throw new IOException("Local " + serviceName + " did not start within " + + timeout + ".", ex); + } catch (ExecutionException ex) { + Throwable cause = ex.getCause(); + throw cause instanceof Exception ? (Exception) cause : ex; + } finally { + executor.shutdownNow(); + } + } + + private void startRecon(OzoneConfiguration conf) throws Exception { + // ReconServer reads its configuration from the static provider, like + // MiniOzoneCluster does when running Recon in the same JVM. + ConfigurationProvider.resetConfiguration(); + ConfigurationProvider.setConfiguration(new OzoneConfiguration(conf)); + // Assign the field before execute() so a failed start can still be rolled + // back by stopServices(). + reconServer = new ReconServer(); + int exitCode = executeWithinStartupTimeout("Recon", + () -> reconServer.execute(NO_ARGS)); + if (exitCode != 0) { + throw new IOException("Failed to start local Recon. Exit code " + + exitCode + "."); + } + } + + private void waitForHttpEndpointReadiness(String endpoint, + String serviceName, Duration timeout) throws Exception { + waitForReadiness(() -> isHttpEndpointReady(endpoint), serviceName, timeout); + } + + private static boolean isHttpEndpointReady(String endpoint) { + HttpURLConnection connection = null; + try { + connection = (HttpURLConnection) new URL(endpoint).openConnection(); + connection.setConnectTimeout((int) READINESS_POLL_INTERVAL_MILLIS); + connection.setReadTimeout((int) READINESS_POLL_INTERVAL_MILLIS); + // Unlike SCM and OM, these services expose no in-process readiness API to this package + // (ReconServer even reports success while its initialization failed); an HTTP response + // from the endpoint is the service-level signal that requests are being served. + connection.getResponseCode(); + return true; + } catch (IOException ex) { + return false; + } finally { + if (connection != null) { + connection.disconnect(); + } + } + } + private void waitForClusterReadiness(Duration timeout) throws Exception { + waitForReadiness(this::isClusterReady, "Ozone cluster", timeout); + } + + private static void waitForReadiness(BooleanSupplier ready, String subject, + Duration timeout) throws InterruptedException, TimeoutException { long deadlineNanos = System.nanoTime() + timeout.toNanos(); - while (true) { - if (isClusterReady()) { - return; - } + while (!ready.getAsBoolean()) { if (System.nanoTime() >= deadlineNanos) { throw new TimeoutException("Timed out waiting " + timeout - + " for the local Ozone cluster to become ready."); + + " for the local " + subject + " to become ready."); } Thread.sleep(READINESS_POLL_INTERVAL_MILLIS); } @@ -745,6 +1020,17 @@ private PersistedPortState loadPersistedPortState() throws IOException { private String[] requiredPersistedPortKeys() { List keys = new ArrayList<>(Arrays.asList(REQUIRED_PERSISTED_PORT_KEYS)); + if (config.isS3gEnabled()) { + keys.add(S3G_HTTP_PORT_KEY); + keys.add(S3G_HTTPS_PORT_KEY); + keys.add(S3G_WEBADMIN_HTTP_PORT_KEY); + keys.add(S3G_WEBADMIN_HTTPS_PORT_KEY); + } + if (config.isReconEnabled()) { + keys.add(RECON_HTTP_PORT_KEY); + keys.add(RECON_HTTPS_PORT_KEY); + keys.add(RECON_DATANODE_PORT_KEY); + } for (int index = 0; index < config.getDatanodes(); index++) { for (String suffix : DATANODE_PORT_KEY_SUFFIXES) { keys.add(datanodePortKey(index, suffix)); @@ -807,14 +1093,19 @@ static final class PreparedConfiguration { private final OzoneConfiguration configuration; private final int scmPort; private final int omPort; + private final int s3gPort; + private final int reconPort; private final List datanodeConfigurations; PreparedConfiguration(OzoneConfiguration configuration, int scmPort, - int omPort, List datanodeConfigurations) { + int omPort, int s3gPort, int reconPort, + List datanodeConfigurations) { this.configuration = Objects.requireNonNull(configuration, "configuration"); this.scmPort = scmPort; this.omPort = omPort; + this.s3gPort = s3gPort; + this.reconPort = reconPort; this.datanodeConfigurations = Collections.unmodifiableList( new ArrayList<>(Objects.requireNonNull(datanodeConfigurations, "datanodeConfigurations"))); @@ -832,6 +1123,14 @@ int getOmPort() { return omPort; } + int getS3gPort() { + return s3gPort; + } + + int getReconPort() { + return reconPort; + } + List getDatanodeConfigurations() { return datanodeConfigurations; } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java index 4a5d9d7ea206..4e2af4e994e4 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java @@ -43,6 +43,7 @@ public final class LocalOzoneClusterConfig { static final String DEFAULT_DATANODES_VALUE = "1"; static final String DEFAULT_PORT_VALUE = "0"; static final String DEFAULT_S3G_ENABLED_VALUE = "true"; + static final String DEFAULT_RECON_ENABLED_VALUE = "false"; static final String DEFAULT_EPHEMERAL_VALUE = "false"; static final String DEFAULT_STARTUP_TIMEOUT_VALUE = "PT2M"; @@ -60,6 +61,8 @@ public final class LocalOzoneClusterConfig { static final int DEFAULT_PORT = Integer.parseInt(DEFAULT_PORT_VALUE); static final boolean DEFAULT_S3G_ENABLED = Boolean.parseBoolean(DEFAULT_S3G_ENABLED_VALUE); + static final boolean DEFAULT_RECON_ENABLED = + Boolean.parseBoolean(DEFAULT_RECON_ENABLED_VALUE); static final boolean DEFAULT_EPHEMERAL = Boolean.parseBoolean(DEFAULT_EPHEMERAL_VALUE); static final Duration DEFAULT_STARTUP_TIMEOUT = @@ -77,6 +80,8 @@ public final class LocalOzoneClusterConfig { private final int omPort; private final int s3gPort; private final boolean s3gEnabled; + private final int reconPort; + private final boolean reconEnabled; private final boolean ephemeral; private final Duration startupTimeout; private final String s3AccessKey; @@ -95,6 +100,8 @@ private LocalOzoneClusterConfig(Builder builder) { omPort = builder.omPort; s3gPort = builder.s3gPort; s3gEnabled = builder.s3gEnabled; + reconPort = builder.reconPort; + reconEnabled = builder.reconEnabled; ephemeral = builder.ephemeral; startupTimeout = Objects.requireNonNull(builder.startupTimeout, "startupTimeout"); @@ -154,6 +161,21 @@ public boolean isS3gEnabled() { return s3gEnabled; } + /** + * Returns the Recon HTTP port. Port {@code 0} asks the runtime to choose an + * available local port. + */ + public int getReconPort() { + return reconPort; + } + + /** + * Returns whether the local runtime should include Recon. + */ + public boolean isReconEnabled() { + return reconEnabled; + } + /** * Returns whether the local runtime should remove its data directory when it * shuts down. @@ -245,6 +267,8 @@ public static final class Builder { private int omPort = DEFAULT_PORT; private int s3gPort = DEFAULT_PORT; private boolean s3gEnabled = DEFAULT_S3G_ENABLED; + private int reconPort = DEFAULT_PORT; + private boolean reconEnabled = DEFAULT_RECON_ENABLED; private boolean ephemeral = DEFAULT_EPHEMERAL; private Duration startupTimeout = DEFAULT_STARTUP_TIMEOUT; private String s3AccessKey = DEFAULT_S3_ACCESS_KEY; @@ -295,6 +319,16 @@ public Builder setS3gEnabled(boolean value) { return this; } + public Builder setReconPort(int value) { + reconPort = value; + return this; + } + + public Builder setReconEnabled(boolean value) { + reconEnabled = value; + return this; + } + public Builder setEphemeral(boolean value) { ephemeral = value; return this; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java index 3ed939044176..b5b03d39773e 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java @@ -71,6 +71,20 @@ public interface LocalOzoneRuntime extends AutoCloseable { */ String getS3Endpoint(); + /** + * Returns the Recon HTTP port for this local runtime. + * + * @return Recon port + */ + int getReconPort(); + + /** + * Returns the full Recon endpoint shown to users. + * + * @return Recon endpoint, including scheme, host, and port + */ + String getReconEndpoint(); + /** * Stops the local runtime and releases resources created during startup. * diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java index 0d5e6ec17345..4add6638f4cc 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java @@ -59,6 +59,8 @@ public class OzoneLocal extends GenericCli { static final String ENV_OM_PORT = "OZONE_LOCAL_OM_PORT"; static final String ENV_S3G_ENABLED = "OZONE_LOCAL_S3G_ENABLED"; static final String ENV_S3G_PORT = "OZONE_LOCAL_S3G_PORT"; + static final String ENV_RECON_ENABLED = "OZONE_LOCAL_RECON_ENABLED"; + static final String ENV_RECON_PORT = "OZONE_LOCAL_RECON_PORT"; static final String ENV_EPHEMERAL = "OZONE_LOCAL_EPHEMERAL"; static final String ENV_STARTUP_TIMEOUT = "OZONE_LOCAL_STARTUP_TIMEOUT"; static final String ENV_S3_ACCESS_KEY = "OZONE_LOCAL_S3_ACCESS_KEY"; @@ -86,6 +88,12 @@ public class OzoneLocal extends GenericCli { + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE + "}"; private static final String DEFAULT_S3G_PORT_VALUE = "${env:" + ENV_S3G_PORT + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + "}"; + private static final String DEFAULT_RECON_ENABLED_VALUE = "${env:" + + ENV_RECON_ENABLED + ":-" + + LocalOzoneClusterConfig.DEFAULT_RECON_ENABLED_VALUE + "}"; + private static final String DEFAULT_RECON_PORT_VALUE = "${env:" + + ENV_RECON_PORT + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + + "}"; private static final String DEFAULT_EPHEMERAL_VALUE = "${env:" + ENV_EPHEMERAL + ":-" + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE + "}"; @@ -114,7 +122,7 @@ public static void main(String[] args) { } @Command(name = "run", - description = "Start SCM, OM, and datanodes in one local process") + description = "Start SCM, OM, datanodes, and optional S3 Gateway and Recon in one local process") static class RunCommand extends AbstractSubcommand implements Callable { @Option(names = "--data-dir", @@ -165,6 +173,18 @@ static class RunCommand extends AbstractSubcommand implements Callable { description = "Enable S3 Gateway") private boolean s3gEnabled; + @Option(names = "--recon-port", + defaultValue = DEFAULT_RECON_PORT_VALUE, + description = "Recon HTTP port (0 means auto-allocate)") + private int reconPort; + + @Option(names = "--recon", + negatable = true, + defaultValue = DEFAULT_RECON_ENABLED_VALUE, + fallbackValue = "true", + description = "Enable Recon") + private boolean reconEnabled; + @Option(names = "--ephemeral", negatable = true, defaultValue = DEFAULT_EPHEMERAL_VALUE, @@ -213,6 +233,19 @@ private void printSummary(LocalOzoneRuntime runtime, LocalOzoneClusterConfig con writer.println("Local Ozone is running from " + config.getDataDir()); writer.println("SCM RPC: " + runtime.getDisplayHost() + ":" + runtime.getScmPort()); writer.println("OM RPC: " + runtime.getDisplayHost() + ":" + runtime.getOmPort()); + if (config.isS3gEnabled()) { + writer.println("S3 endpoint: " + runtime.getS3Endpoint()); + writer.println("Suggested local AWS settings:"); + writer.println("AWS_ACCESS_KEY_ID=" + config.getS3AccessKey()); + writer.println("AWS_SECRET_ACCESS_KEY=" + config.getS3SecretKey()); + writer.println("AWS_REGION=" + config.getS3Region()); + writer.println("AWS_ENDPOINT_URL_S3=" + runtime.getS3Endpoint()); + writer.println("AWS_S3_FORCE_PATH_STYLE=true"); + writer.println("These credentials are pre-provisioned for the local S3 Gateway."); + } + if (config.isReconEnabled()) { + writer.println("Recon endpoint: " + runtime.getReconEndpoint()); + } writer.println("Press Ctrl+C to stop."); writer.flush(); } @@ -252,6 +285,7 @@ LocalOzoneClusterConfig resolveConfig() { validatePort(scmPort, "--scm-port"); validatePort(omPort, "--om-port"); validatePort(s3gPort, "--s3g-port"); + validatePort(reconPort, "--recon-port"); validateStartupTimeout(); return LocalOzoneClusterConfig.builder(dataDir) @@ -263,6 +297,8 @@ LocalOzoneClusterConfig resolveConfig() { .setOmPort(omPort) .setS3gPort(s3gPort) .setS3gEnabled(s3gEnabled) + .setReconPort(reconPort) + .setReconEnabled(reconEnabled) .setEphemeral(ephemeral) .setStartupTimeout(startupTimeout) .setS3AccessKey(s3AccessKey) diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java index 1f109c6de420..6a1e47ae558d 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; @@ -31,6 +32,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -42,7 +47,12 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashSet; import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -74,9 +84,29 @@ void prepareConfigurationExposesPreparedPorts() throws Exception { assertTrue(Files.isRegularFile(portStateFile(dataDir))); assertTrue(prepared.getScmPort() > 0); assertTrue(prepared.getOmPort() > 0); + assertTrue(cluster.getS3gPort() > 0); + assertEquals("http://127.0.0.1:" + cluster.getS3gPort(), + cluster.getS3Endpoint()); + } + } + + @Test + void prepareConfigurationSkipsS3GatewayWhenDisabled() throws Exception { + Path dataDir = tempDir.resolve("local-ozone"); + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir) + .setS3gEnabled(false) + .build(); + + try (LocalOzoneCluster cluster = newCluster(config)) { + LocalOzoneCluster.PreparedConfiguration prepared = + cluster.prepareConfiguration(); + + assertEquals(-1, prepared.getS3gPort()); assertEquals(-1, cluster.getS3gPort()); assertEquals("", cluster.getS3Endpoint()); } + assertFalse(loadPortState(dataDir).stringPropertyNames().stream() + .anyMatch(key -> key.startsWith("s3g."))); } @Test @@ -86,6 +116,7 @@ void prepareConfigurationCreatesBaseLayoutAndLocalDefaults() LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir) .setScmPort(9860) .setOmPort(9862) + .setS3gPort(9878) .setDatanodes(2) .build(); @@ -107,10 +138,31 @@ void prepareConfigurationCreatesBaseLayoutAndLocalDefaults() assertEquals(2, conf.getInt(HDDS_SCM_SAFEMODE_MIN_DATANODE, 0)); assertTrue(conf.get(OZONE_SCM_CLIENT_ADDRESS_KEY).endsWith(":9860")); assertTrue(conf.get(OZONE_OM_ADDRESS_KEY).endsWith(":9862")); + assertTrue(conf.get(OZONE_S3G_HTTP_ADDRESS_KEY).endsWith(":9878")); assertTrue(conf.getTrimmedStringCollection(OZONE_SCM_NAMES).iterator() .next().contains(":")); assertEquals(9860, prepared.getScmPort()); assertEquals(9862, prepared.getOmPort()); + assertEquals(9878, prepared.getS3gPort()); + } + + @Test + void prepareConfigurationAllocatesDistinctS3GatewayPorts() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + tempDir.resolve("local-ozone")) + .setS3gPort(9878) + .build(); + + OzoneConfiguration conf = prepare(config).getConfiguration(); + + int httpPort = parsePort(conf.get(OZONE_S3G_HTTP_ADDRESS_KEY)); + int httpsPort = parsePort(conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY)); + int webHttpPort = parsePort(conf.get(OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY)); + int webHttpsPort = parsePort(conf.get(OZONE_S3G_WEBADMIN_HTTPS_ADDRESS_KEY)); + + assertEquals(9878, httpPort); + assertEquals(4, new HashSet<>(Arrays.asList( + httpPort, httpsPort, webHttpPort, webHttpsPort)).size()); } @Test @@ -303,10 +355,86 @@ void persistedPortFileContainsDistinctAllocatedPorts() throws Exception { assertPositivePort(properties, "om.rpc"); assertPositivePort(properties, "om.http"); assertPositivePort(properties, "om.ratis"); + assertPositivePort(properties, "s3g.http"); + assertPositivePort(properties, "s3g.https"); + assertPositivePort(properties, "s3g.web.http"); + assertPositivePort(properties, "s3g.web.https"); assertNotEquals(properties.getProperty("scm.client"), properties.getProperty("om.rpc")); } + @Test + void prepareConfigurationReservesReconPortsWhenEnabled() throws Exception { + Path dataDir = tempDir.resolve("local-ozone"); + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir) + .setReconEnabled(true) + .setReconPort(9888) + .build(); + + try (LocalOzoneCluster cluster = newCluster(config)) { + LocalOzoneCluster.PreparedConfiguration prepared = + cluster.prepareConfiguration(); + OzoneConfiguration conf = prepared.getConfiguration(); + + assertEquals(9888, prepared.getReconPort()); + assertEquals(9888, cluster.getReconPort()); + assertEquals("http://127.0.0.1:9888", cluster.getReconEndpoint()); + assertTrue(conf.get(OZONE_RECON_HTTP_ADDRESS_KEY).endsWith(":9888")); + assertTrue(Files.isDirectory(dataDir.resolve("recon"))); + } + Properties properties = loadPortState(dataDir); + assertPositivePort(properties, "recon.http"); + assertPositivePort(properties, "recon.https"); + assertPositivePort(properties, "recon.datanode"); + } + + @Test + void prepareConfigurationSkipsReconWhenDisabled() throws Exception { + Path dataDir = tempDir.resolve("local-ozone"); + LocalOzoneClusterConfig config = + LocalOzoneClusterConfig.builder(dataDir).build(); + + try (LocalOzoneCluster cluster = newCluster(config)) { + LocalOzoneCluster.PreparedConfiguration prepared = + cluster.prepareConfiguration(); + + assertEquals(-1, prepared.getReconPort()); + assertEquals(-1, cluster.getReconPort()); + assertEquals("", cluster.getReconEndpoint()); + } + assertFalse(loadPortState(dataDir).stringPropertyNames().stream() + .anyMatch(key -> key.startsWith("recon."))); + } + + @Test + void formatNeverRejectsPortStateMissingReconPorts() throws Exception { + Path dataDir = tempDir.resolve("local-ozone"); + prepare(LocalOzoneClusterConfig.builder(dataDir).build()); + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir) + .setFormatMode(LocalOzoneClusterConfig.FormatMode.NEVER) + .setReconEnabled(true) + .build(); + + IOException error = assertPrepareFails(config); + + assertMessageContains(error, "recon."); + } + + @Test + void formatNeverRejectsPortStateMissingS3GatewayPorts() throws Exception { + Path dataDir = tempDir.resolve("local-ozone"); + prepare(LocalOzoneClusterConfig.builder(dataDir) + .setS3gEnabled(false) + .build()); + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir) + .setFormatMode(LocalOzoneClusterConfig.FormatMode.NEVER) + .build(); + + IOException error = assertPrepareFails(config); + + assertMessageContains(error, "s3g."); + } + @Test void prepareConfigurationCreatesDatanodeConfigurations() throws Exception { Path dataDir = tempDir.resolve("local-ozone"); @@ -414,6 +542,65 @@ void getDatanodeCountReturnsZeroBeforeStart() throws Exception { } } + @Test + void executeWithinStartupTimeoutReturnsStartupResult() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + tempDir.resolve("local-ozone")) + .setStartupTimeout(Duration.ofSeconds(30)) + .build(); + + try (LocalOzoneCluster cluster = newCluster(config)) { + assertEquals(0, cluster.executeWithinStartupTimeout("Recon", () -> 0)); + } + } + + @Test + void executeWithinStartupTimeoutFailsFastWhenStartupBlocks() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + tempDir.resolve("local-ozone")) + .setStartupTimeout(Duration.ofMillis(200)) + .build(); + CountDownLatch interrupted = new CountDownLatch(1); + + try (LocalOzoneCluster cluster = newCluster(config)) { + IOException error = + assertThrows(IOException.class, () -> cluster.executeWithinStartupTimeout( + "Recon", () -> { + try { + // Simulate an in-JVM startup that blocks until it leaves safe + // mode; the deadline must interrupt it rather than hang. + Thread.sleep(TimeUnit.MINUTES.toMillis(5)); + } catch (InterruptedException e) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + return 0; + })); + + assertMessageContains(error, "did not start within"); + assertTrue(interrupted.await(30, TimeUnit.SECONDS), + "blocked startup should be interrupted on timeout"); + } + } + + @Test + void executeWithinStartupTimeoutPropagatesStartupFailure() throws Exception { + LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder( + tempDir.resolve("local-ozone")) + .setStartupTimeout(Duration.ofSeconds(30)) + .build(); + + try (LocalOzoneCluster cluster = newCluster(config)) { + IOException error = + assertThrows(IOException.class, () -> cluster.executeWithinStartupTimeout( + "Recon", () -> { + throw new IOException("startup boom"); + })); + + assertMessageContains(error, "startup boom"); + } + } + private LocalOzoneCluster.PreparedConfiguration prepare( LocalOzoneClusterConfig config) throws IOException { try (LocalOzoneCluster cluster = newCluster(config)) { @@ -456,6 +643,10 @@ private void assertPositivePort(Properties properties, String key) { assertTrue(Integer.parseInt(properties.getProperty(key)) > 0, key); } + private static int parsePort(String address) { + return Integer.parseInt(address.substring(address.lastIndexOf(':') + 1)); + } + private Path metadataDir(Path dataDir) { return dataDir.resolve(METADATA_DIR_NAME); } diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java index 7e7f82480246..3fee55f46d9c 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java @@ -46,6 +46,8 @@ void builderProvidesLocalClusterDefaults() { assertEquals(0, config.getOmPort()); assertEquals(0, config.getS3gPort()); assertTrue(config.isS3gEnabled()); + assertEquals(0, config.getReconPort()); + assertFalse(config.isReconEnabled()); assertFalse(config.isEphemeral()); assertEquals(Duration.ofMinutes(2), config.getStartupTimeout()); assertEquals("admin", config.getS3AccessKey()); @@ -66,6 +68,9 @@ void typedDefaultsMatchSharedFallbackValues() { assertEquals(Boolean.parseBoolean( LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE), LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED); + assertEquals(Boolean.parseBoolean( + LocalOzoneClusterConfig.DEFAULT_RECON_ENABLED_VALUE), + LocalOzoneClusterConfig.DEFAULT_RECON_ENABLED); assertEquals(Boolean.parseBoolean( LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE), LocalOzoneClusterConfig.DEFAULT_EPHEMERAL); @@ -86,6 +91,8 @@ void builderAcceptsExplicitOverrides() { .setOmPort(9862) .setS3gPort(9878) .setS3gEnabled(false) + .setReconPort(9888) + .setReconEnabled(true) .setEphemeral(true) .setStartupTimeout(Duration.ofSeconds(45)) .setS3AccessKey("dev") @@ -104,6 +111,8 @@ void builderAcceptsExplicitOverrides() { assertEquals(9862, config.getOmPort()); assertEquals(9878, config.getS3gPort()); assertFalse(config.isS3gEnabled()); + assertEquals(9888, config.getReconPort()); + assertTrue(config.isReconEnabled()); assertTrue(config.isEphemeral()); assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); assertEquals("dev", config.getS3AccessKey()); diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java index 0f8686e91a2b..a91e711fb8ce 100644 --- a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java @@ -93,7 +93,8 @@ void rootHelpListsRunCommand() throws Exception { @Test void runCommandStartsRuntimeAndPrintsStartupSummary() throws Exception { ByteArrayOutputStream out = new ByteArrayOutputStream(); - StubRuntime runtime = new StubRuntime("localhost", 9860, 9862); + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, + "http://localhost:9878"); TestableRunCommand command = new TestableRunCommand(runtime); CommandLine commandLine = new CommandLine(command); commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), @@ -108,12 +109,74 @@ void runCommandStartsRuntimeAndPrintsStartupSummary() throws Exception { assertTrue(text.contains("Local Ozone is running from"), text); assertTrue(text.contains("SCM RPC: localhost:9860"), text); assertTrue(text.contains("OM RPC: localhost:9862"), text); + assertTrue(text.contains("S3 endpoint: http://localhost:9878"), text); + assertTrue(text.contains("AWS_ACCESS_KEY_ID=" + + LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY), text); + assertTrue(text.contains("AWS_SECRET_ACCESS_KEY=" + + LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY), text); + assertTrue(text.contains("AWS_REGION=" + + LocalOzoneClusterConfig.DEFAULT_S3_REGION), text); + assertTrue(text.contains("AWS_ENDPOINT_URL_S3=http://localhost:9878"), + text); + assertTrue(text.contains("AWS_S3_FORCE_PATH_STYLE=true"), text); + assertTrue(text.contains("Press Ctrl+C to stop."), text); + } + + @Test + void runCommandPrintsReconEndpointWhenEnabled() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, + "http://localhost:9878"); + runtime.reconEndpoint = "http://localhost:9888"; + TestableRunCommand command = new TestableRunCommand(runtime); + CommandLine commandLine = new CommandLine(command); + commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), + true)); + + int exitCode = commandLine.execute("--recon"); + + assertEquals(0, exitCode); + String text = out.toString(UTF_8.name()); + assertTrue(text.contains("Recon endpoint: http://localhost:9888"), text); + } + + @Test + void runCommandOmitsReconEndpointWhenDisabled() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, + "http://localhost:9878"); + TestableRunCommand command = new TestableRunCommand(runtime); + CommandLine commandLine = new CommandLine(command); + commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), + true)); + + int exitCode = commandLine.execute(); + + assertEquals(0, exitCode); + assertFalse(out.toString(UTF_8.name()).contains("Recon endpoint:")); + } + + @Test + void runCommandOmitsS3SummaryWhenS3gDisabled() throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, ""); + TestableRunCommand command = new TestableRunCommand(runtime); + CommandLine commandLine = new CommandLine(command); + commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8), + true)); + + int exitCode = commandLine.execute("--no-s3g"); + + assertEquals(0, exitCode); + String text = out.toString(UTF_8.name()); + assertFalse(text.contains("S3 endpoint:"), text); + assertFalse(text.contains("AWS_ACCESS_KEY_ID="), text); assertTrue(text.contains("Press Ctrl+C to stop."), text); } @Test void runCommandClosesRuntimeWhenStartupFails() { - StubRuntime runtime = new StubRuntime("localhost", 9860, 9862); + StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, ""); runtime.failStart = true; TestableRunCommand command = new TestableRunCommand(runtime); @@ -143,6 +206,10 @@ void runCommandOptionsUseEnvironmentDefaults() throws Exception { LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE); assertEnvDefault("s3gPort", OzoneLocal.ENV_S3G_PORT, LocalOzoneClusterConfig.DEFAULT_PORT_VALUE); + assertEnvDefault("reconEnabled", OzoneLocal.ENV_RECON_ENABLED, + LocalOzoneClusterConfig.DEFAULT_RECON_ENABLED_VALUE); + assertEnvDefault("reconPort", OzoneLocal.ENV_RECON_PORT, + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE); assertEnvDefault("ephemeral", OzoneLocal.ENV_EPHEMERAL, LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE); assertEnvDefault("startupTimeout", OzoneLocal.ENV_STARTUP_TIMEOUT, @@ -170,6 +237,8 @@ void resolveConfigUsesPicocliDefaults() { assertEquals(0, config.getOmPort()); assertEquals(0, config.getS3gPort()); assertTrue(config.isS3gEnabled()); + assertEquals(0, config.getReconPort()); + assertFalse(config.isReconEnabled()); assertFalse(config.isEphemeral()); assertEquals(Duration.ofMinutes(2), config.getStartupTimeout()); assertEquals("admin", config.getS3AccessKey()); @@ -189,6 +258,8 @@ void resolveConfigUsesCliOverrides() { "--om-port", "201", "--s3g-port", "202", "--no-s3g", + "--recon-port", "203", + "--recon", "--ephemeral", "--startup-timeout", "45s", "--s3-access-key", "cli-access", @@ -206,6 +277,8 @@ void resolveConfigUsesCliOverrides() { assertEquals(201, config.getOmPort()); assertEquals(202, config.getS3gPort()); assertFalse(config.isS3gEnabled()); + assertEquals(203, config.getReconPort()); + assertTrue(config.isReconEnabled()); assertTrue(config.isEphemeral()); assertEquals(Duration.ofSeconds(45), config.getStartupTimeout()); assertEquals("cli-access", config.getS3AccessKey()); @@ -228,6 +301,18 @@ void resolveConfigAllowsS3gAndEphemeralToBeNegated() { assertFalse(config.isEphemeral()); } + @Test + void resolveConfigAllowsReconToBeNegated() { + LocalOzoneClusterConfig config = resolve("--no-recon"); + + assertFalse(config.isReconEnabled()); + } + + @Test + void resolveConfigRejectsInvalidReconPort() { + assertConfigError("--recon-port", "65536", "--recon-port"); + } + @Test void resolveConfigRejectsInvalidFormat() { assertParseError("--format", "sometimes", "--format"); @@ -362,14 +447,18 @@ private static final class StubRuntime implements LocalOzoneRuntime { private final String displayHost; private final int scmPort; private final int omPort; + private final String s3Endpoint; + private String reconEndpoint = ""; private boolean failStart; private boolean started; private boolean closed; - private StubRuntime(String displayHost, int scmPort, int omPort) { + private StubRuntime(String displayHost, int scmPort, int omPort, + String s3Endpoint) { this.displayHost = displayHost; this.scmPort = scmPort; this.omPort = omPort; + this.s3Endpoint = s3Endpoint; } @Override @@ -402,7 +491,17 @@ public int getS3gPort() { @Override public String getS3Endpoint() { - return ""; + return s3Endpoint; + } + + @Override + public int getReconPort() { + return 0; + } + + @Override + public String getReconEndpoint() { + return reconEndpoint; } @Override @@ -434,8 +533,12 @@ public String defaultValue(ArgSpec argSpec) { || "--om-port".equals(option) || "--s3g-port".equals(option)) { return LocalOzoneClusterConfig.DEFAULT_PORT_VALUE; + } else if ("--recon-port".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_PORT_VALUE; } else if ("--s3g".equals(option)) { return LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE; + } else if ("--recon".equals(option)) { + return LocalOzoneClusterConfig.DEFAULT_RECON_ENABLED_VALUE; } else if ("--ephemeral".equals(option)) { return LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE; } else if ("--startup-timeout".equals(option)) {