HDDS-14932. Optional Recon and Httpfs Start Up

hadoop-hdds/docs/content/tools/_index.md

@@ -43,8 +43,8 @@ Client commands:
 
    * **sh** -  Primary command line interface for ozone to manage volumes/buckets/keys.
    * **fs** - Runs a command on ozone file system (similar to `hdfs dfs`)
-   * **local** - Runs a single-node local Ozone cluster (SCM, OM, and datanodes)
-   in one process for development.
+   * **local** - Runs a single-node local Ozone cluster (SCM, OM, datanodes,
+   and optional S3 Gateway and Recon) in one process for development.
    * **version** - Prints the version of Ozone and HDDS.
 
 

hadoop-ozone/integration-test-recon/pom.xml

@@ -180,6 +180,11 @@
       <artifactId>ozone-reconcodegen</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-tools</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-common</artifactId>

hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneRecon.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.local;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Path;
+import java.time.Duration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Integration tests for Recon in the {@code ozone local} runtime.
+ */
+class TestLocalOzoneRecon {
+
+  @TempDir
+  private Path tempDir;
+
+  @Test
+  void reconServesRequestsWhenEnabled() throws Exception {
+    LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
+            tempDir.resolve("local-ozone-recon"))
+        .setS3gEnabled(false)
+        .setReconEnabled(true)
+        .setStartupTimeout(Duration.ofMinutes(2))
+        .build();
+
+    try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, new OzoneConfiguration())) {
+      cluster.start();
+
+      assertTrue(cluster.getReconPort() > 0);
+      assertHttpEndpointResponds(cluster.getReconEndpoint());
+    }
+  }
+
+  private static void assertHttpEndpointResponds(String endpoint) throws Exception {
+    HttpURLConnection connection = (HttpURLConnection) new URL(endpoint).openConnection();
+    try {
+      connection.setConnectTimeout(1_000);
+      connection.setReadTimeout(1_000);
+      // Any HTTP response proves Recon is serving requests.
+      assertTrue(connection.getResponseCode() > 0, endpoint);
+    } finally {
+      connection.disconnect();
+    }
+  }
+}

hadoop-ozone/integration-test-s3/pom.xml

@@ -154,6 +154,11 @@
       <artifactId>ozone-s3gateway</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-tools</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-common</artifactId>

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.local;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Path;
+import java.time.Duration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Integration tests for the S3 Gateway of the {@code ozone local} runtime.
+ */
+class TestLocalOzoneS3 {
+
+  @TempDir
+  private Path tempDir;
+
+  @Test
+  void s3GatewayServesRequests() throws Exception {
+    LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
+            tempDir.resolve("local-ozone-s3"))
+        .setStartupTimeout(Duration.ofMinutes(2))
+        .build();
+
+    try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, new OzoneConfiguration())) {
+      cluster.start();
+
+      assertTrue(cluster.getS3gPort() > 0);
+      assertS3EndpointResponds(cluster.getS3Endpoint());
+    }
+  }
+
+  private static void assertS3EndpointResponds(String endpoint) throws Exception {
+    HttpURLConnection connection = (HttpURLConnection) new URL(endpoint).openConnection();
+    try {
+      connection.setConnectTimeout(1_000);
+      connection.setReadTimeout(1_000);
+      // Any HTTP response proves the S3 Gateway is serving requests.
+      assertTrue(connection.getResponseCode() > 0, endpoint);
+    } finally {
+      connection.disconnect();
+    }
+  }
+}

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ConfigurationProvider.java

@@ -17,7 +17,6 @@
 
 package org.apache.hadoop.ozone.recon;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.inject.Provider;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration.DeprecationDelta;
@@ -50,16 +49,14 @@ private static void addDeprecations() {
     });
   }
 
-  @VisibleForTesting
   public static void setConfiguration(OzoneConfiguration conf) {
-    // Nullity check is used in case the configuration was already set
-    // in the MiniOzoneCluster
+    // Nullity check is used in case the configuration was already set by a
+    // same-JVM launcher (MiniOzoneCluster or ozone local)
     if (configuration == null) {
       ConfigurationProvider.configuration = conf;
     }
   }
 
-  @VisibleForTesting
   public static void resetConfiguration() {
     configuration = null;
   }

hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java

@@ -17,7 +17,6 @@
 
 package org.apache.hadoop.ozone.s3;
 
-import com.google.common.annotations.VisibleForTesting;
 import javax.enterprise.inject.Produces;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 
@@ -39,17 +38,15 @@ public static OzoneConfiguration configuration() {
     return configuration;
   }
 
-  @VisibleForTesting
   public static void setConfiguration(
       OzoneConfiguration conf) {
-    // Nullity check is used in case the configuration was already set
-    // in the MiniOzoneCluster
+    // Nullity check is used in case the configuration was already set by a
+    // same-JVM launcher (MiniOzoneCluster or ozone local)
     if (configuration == null) {
       OzoneConfigurationHolder.configuration = conf;
     }
   }
 
-  @VisibleForTesting
   public static void resetConfiguration() {
     configuration = null;
   }

hadoop-ozone/tools/pom.xml

@@ -82,6 +82,14 @@
       <groupId>org.apache.ozone</groupId>
       <artifactId>ozone-manager</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-recon</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-s3gateway</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-common</artifactId>