HDDS-15090. Add Compose and Docs

hadoop-hdds/docs/content/tools/_index.md

@@ -43,8 +43,8 @@ Client commands:
 
    * **sh** -  Primary command line interface for ozone to manage volumes/buckets/keys.
    * **fs** - Runs a command on ozone file system (similar to `hdfs dfs`)
-   * **local** - Runs a single-node local Ozone cluster (SCM, OM, and datanodes)
-   in one process for development.
+   * **local** - Runs a single-node local Ozone cluster (SCM, OM, datanodes,
+   and optional S3 Gateway and Recon) in one process for development.
    * **version** - Prints the version of Ozone and HDDS.
 
 

hadoop-ozone/dist/src/main/compose/README.md

@@ -36,6 +36,10 @@ Most environments allow scaling the number of datanodes.
 
 Key ports (web UI, RPC) are usually published on the docker host.  Datanode ports are published on random local ports, since there can be multiple instances of the same service.
 
+The [`ozone-local/`](./ozone-local/) example is the single-container option. It
+runs the packaged `ozone local run` command and configures the cluster only with
+CLI flags and `OZONE_LOCAL_*` environment variables in Compose.
+
 ### Starting the Cluster
 
 ```

hadoop-ozone/dist/src/main/compose/ozone-local/.env

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+OZONE_RUNNER_VERSION=${docker.ozone-runner.version}
+OZONE_RUNNER_IMAGE=apache/ozone-runner

hadoop-ozone/dist/src/main/compose/ozone-local/README.md

@@ -0,0 +1,64 @@
+<!---
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+# Ozone Local Compose Example
+
+This Compose definition runs the packaged `ozone local run` command in a single
+container. The quickstart surface is intentionally small: the Compose file only
+sets the local S3 credentials and pins the published ports, then lets
+`ozone local run` use its defaults for the single-node cluster.
+
+## Usage
+
+Start the container:
+
+```bash
+docker-compose up -d
+docker-compose logs -f local
+```
+
+The startup summary prints the suggested local AWS settings. The default S3
+endpoint from this example is `http://127.0.0.1:9878`. Recon is also started
+by `ozone local run` and is available at `http://127.0.0.1:9888`.
+
+Example AWS CLI invocation from the host:
+
+```bash
+AWS_ACCESS_KEY_ID=admin \
+AWS_SECRET_ACCESS_KEY=admin123 \
+AWS_REGION=us-east-1 \
+aws --endpoint-url http://127.0.0.1:9878 s3 ls
+```
+
+Stop and remove the container and its named volume:
+
+```bash
+docker-compose down -v
+```
+
+## Advanced configuration
+
+The Compose file pins the S3 Gateway and Recon ports with
+`ozone local run --s3g-port 9878 --recon --recon-port 9888` so the host can
+publish stable local endpoints. Additional local runtime settings can be
+provided with `OZONE_LOCAL_*` environment variables, for example:
+
+```yaml
+environment:
+  OZONE_LOCAL_S3_ACCESS_KEY: admin
+  OZONE_LOCAL_S3_SECRET_KEY: admin123
+  OZONE_LOCAL_DATANODES: 2
+  OZONE_LOCAL_FORMAT: always
+  OZONE_LOCAL_STARTUP_TIMEOUT: PT3M
+```

hadoop-ozone/dist/src/main/compose/ozone-local/docker-compose.yaml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  local:
+    image: ${OZONE_RUNNER_IMAGE}:${OZONE_RUNNER_VERSION}
+    user: "0:0"
+    volumes:
+      - ../..:/opt/hadoop
+      - ozone-local-data:/root/.ozone/local
+    environment:
+      OZONE_LOCAL_S3_ACCESS_KEY: admin
+      OZONE_LOCAL_S3_SECRET_KEY: admin123
+    ports:
+      - 9878:9878
+      - 9888:9888
+    command:
+      - ozone
+      - local
+      - run
+      - --s3g-port
+      - "9878"
+      - --recon
+      - --recon-port
+      - "9888"
+
+volumes:
+  ozone-local-data:

hadoop-ozone/dist/src/main/compose/ozone-local/test.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#suite:unsecure
+
+set -u -o pipefail
+
+COMPOSE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+export COMPOSE_DIR
+
+export SCM=local
+export SECURITY_ENABLED=false
+
+# shellcheck source=/dev/null
+source "$COMPOSE_DIR/../testlib.sh"
+
+# The single ozone local container has no separate SCM/OM services; the
+# published S3 Gateway and Recon ports signal that the runtime is ready.
+wait_for_safemode_exit() {
+  wait_for_port 127.0.0.1 9878 180
+  wait_for_port 127.0.0.1 9888 180
+}
+
+wait_for_om_leader() {
+  return 0
+}
+
+start_docker_env 1
+
+execute_robot_test local ozonelocal/locals3.robot

hadoop-ozone/dist/src/main/smoketest/cli/classpath.robot

@@ -21,19 +21,31 @@ Resource            ../ozone-lib/shell.robot
 Test Timeout        5 minutes
 
 *** Variables ***
-${TEMP_DIR}    /tmp
+${TEMP_DIR}                    /tmp
+@{OZONE_INSTALL_COPY_DIRS}     bin    etc    libexec    share
 
 *** Keywords ***
 Append missing jar path to classpath descriptor
     [arguments]    ${ozone_home}    ${artifact}    ${bogus_jar}
     ${cp_file} =    Set Variable    ${ozone_home}/share/ozone/classpath/${artifact}.classpath
     Execute    sed -i 's_$_:${bogus_jar}_' '${cp_file}'
 
+Copy Ozone install for classpath test
+    # Copy only the directories the classpath scripts need, so data left in the
+    # install dir by other acceptance environments (e.g. compose volumes owned
+    # by another user) cannot break the copy.
+    [arguments]    ${ozone_copy}
+    Run Keyword And Ignore Error    Remove Directory    ${ozone_copy}    recursive=True
+    Create Directory    ${ozone_copy}
+    FOR    ${entry}    IN    @{OZONE_INSTALL_COPY_DIRS}
+        Copy Directory    ${OZONE_DIR}/${entry}    ${ozone_copy}
+    END
+
 Copy Ozone install and inject missing jar classpath entry
     [arguments]    ${copy_subdir}    ${bogus_jar_basename}    ${artifact}
     ${OZONE_COPY} =    Set Variable    ${TEMP_DIR}/${copy_subdir}
     ${bogus_jar} =    Set Variable    ${TEMP_DIR}/${bogus_jar_basename}
-    Copy Directory    ${OZONE_DIR}    ${OZONE_COPY}
+    Copy Ozone install for classpath test    ${OZONE_COPY}
     Append missing jar path to classpath descriptor    ${OZONE_COPY}    ${artifact}    ${bogus_jar}
     [return]    ${OZONE_COPY}    ${bogus_jar}
 
@@ -57,7 +69,7 @@ Picks up items from OZONE_CLASSPATH
 Adds optional dir entries
     Set Environment Variable   OZONE_CLASSPATH  ${EMPTY}
     ${OZONE_COPY} =     Set Variable     ${TEMP_DIR}/ozone-copy
-    Copy Directory      ${OZONE_DIR}     ${OZONE_COPY}
+    Copy Ozone install for classpath test    ${OZONE_COPY}
     ${jars_dir} =       Find Jars Dir    ${OZONE_COPY}
     Create File         ${jars_dir}/ozone-insight/optional.jar
 

hadoop-ozone/dist/src/main/smoketest/ozonelocal/locals3.robot

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Minimal AWS CLI smoke for packaged ozone local S3.
+Library             OperatingSystem
+Library             String
+Resource            ../commonlib.robot
+Resource            ../s3/commonawslib.robot
+Test Timeout        5 minutes
+Suite Setup         Setup local ozone S3 smoke
+
+*** Variables ***
+${ENDPOINT_URL}       http://127.0.0.1:9878
+${S3_ACCESS_KEY}      admin
+${S3_SECRET_KEY}      admin123
+${S3_REGION}          us-east-1
+
+*** Keywords ***
+Setup local ozone S3 smoke
+    Install aws cli
+    # The launcher pre-provisions these credentials for the local S3 Gateway.
+    Execute                    aws configure set aws_access_key_id ${S3_ACCESS_KEY}
+    Execute                    aws configure set aws_secret_access_key ${S3_SECRET_KEY}
+    Execute                    aws configure set region ${S3_REGION}
+    Execute                    aws configure set default.s3.signature_version s3v4
+    Execute                    aws configure set default.s3.addressing_style path
+    ${random} =                Generate Random String    8    [LOWER]
+    Set Suite Variable         ${BUCKET}    bucket-${random}
+    Create bucket with name    ${BUCKET}
+
+*** Test Cases ***
+AWS CLI can create list put and get against ozone local
+    Execute                    echo "local ozone" > /tmp/local-ozone-smoke.txt
+    ${result} =                Execute AWSS3APICli    list-buckets
+    Should Contain             ${result}    ${BUCKET}
+    ${result} =                Execute AWSS3Cli    cp /tmp/local-ozone-smoke.txt s3://${BUCKET}/smoke.txt
+    Should Contain             ${result}    upload
+    ${result} =                Execute AWSS3Cli    ls s3://${BUCKET}
+    Should Contain             ${result}    smoke.txt
+    Execute AWSS3APICli        get-object --bucket ${BUCKET} --key smoke.txt /tmp/local-ozone-smoke-copy.txt
+    Compare files              /tmp/local-ozone-smoke.txt    /tmp/local-ozone-smoke-copy.txt

hadoop-ozone/integration-test-recon/pom.xml

@@ -180,6 +180,11 @@
       <artifactId>ozone-reconcodegen</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-tools</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-common</artifactId>

hadoop-ozone/integration-test-recon/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneRecon.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.local;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.file.Path;
+import java.time.Duration;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+/**
+ * Integration tests for Recon in the {@code ozone local} runtime.
+ */
+class TestLocalOzoneRecon {
+
+  @TempDir
+  private Path tempDir;
+
+  @Test
+  void reconServesRequestsWhenEnabled() throws Exception {
+    LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
+            tempDir.resolve("local-ozone-recon"))
+        .setS3gEnabled(false)
+        .setReconEnabled(true)
+        .setStartupTimeout(Duration.ofMinutes(2))
+        .build();
+
+    try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, new OzoneConfiguration())) {
+      cluster.start();
+
+      assertTrue(cluster.getReconPort() > 0);
+      assertHttpEndpointResponds(cluster.getReconEndpoint());
+    }
+  }
+
+  private static void assertHttpEndpointResponds(String endpoint) throws Exception {
+    HttpURLConnection connection = (HttpURLConnection) new URL(endpoint).openConnection();
+    try {
+      connection.setConnectTimeout(1_000);
+      connection.setReadTimeout(1_000);
+      // Any HTTP response proves Recon is serving requests.
+      assertTrue(connection.getResponseCode() > 0, endpoint);
+    } finally {
+      connection.disconnect();
+    }
+  }
+}

hadoop-ozone/integration-test-s3/pom.xml

@@ -154,6 +154,11 @@
       <artifactId>ozone-s3gateway</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.ozone</groupId>
+      <artifactId>ozone-tools</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-common</artifactId>