Skip to content

Commit d567498

Browse files
henrybear327henrybear327
committed
HDDS-15088. Add S3 SDK test suite and smoke test for ozone local
1 parent b60dd0d commit d567498

2 files changed

Lines changed: 103 additions & 0 deletions

File tree

‎hadoop-ozone/dist/src/main/smoketest/ozonelocal/locals3.robot‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
*** Settings ***
17+
Documentation Minimal AWS CLI smoke for packaged ozone local S3.
18+
Library OperatingSystem
19+
Library String
20+
Resource ../commonlib.robot
21+
Resource ../s3/commonawslib.robot
22+
Test Timeout 5 minutes
23+
Suite Setup Setup local ozone S3 smoke
24+
25+
*** Variables ***
26+
${ENDPOINT_URL} http://127.0.0.1:9878
27+
${S3_ACCESS_KEY} admin
28+
${S3_SECRET_KEY} admin123
29+
30+
*** Keywords ***
31+
Setup local ozone S3 smoke
32+
Install aws cli
33+
# The launcher pre-provisions these credentials for the local S3 Gateway.
34+
Execute aws configure set aws_access_key_id ${S3_ACCESS_KEY}
35+
Execute aws configure set aws_secret_access_key ${S3_SECRET_KEY}
36+
Execute aws configure set default.s3.signature_version s3v4
37+
Execute aws configure set default.s3.addressing_style path
38+
${random} = Generate Random String 8 [LOWER]
39+
Set Suite Variable ${BUCKET} bucket-${random}
40+
Create bucket with name ${BUCKET}
41+
42+
*** Test Cases ***
43+
AWS CLI can create list put and get against ozone local
44+
Execute echo "local ozone" > /tmp/local-ozone-smoke.txt
45+
${result} = Execute AWSS3APICli list-buckets
46+
Should Contain ${result} ${BUCKET}
47+
${result} = Execute AWSS3Cli cp /tmp/local-ozone-smoke.txt s3://${BUCKET}/smoke.txt
48+
Should Contain ${result} upload
49+
${result} = Execute AWSS3Cli ls s3://${BUCKET}
50+
Should Contain ${result} smoke.txt
51+
Execute AWSS3APICli get-object --bucket ${BUCKET} --key smoke.txt /tmp/local-ozone-smoke-copy.txt
52+
Compare files /tmp/local-ozone-smoke.txt /tmp/local-ozone-smoke-copy.txt

‎hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,26 @@
1717

1818
package org.apache.hadoop.ozone.local;
1919

20+
import static org.junit.jupiter.api.Assertions.assertEquals;
2021
import static org.junit.jupiter.api.Assertions.assertTrue;
2122

2223
import java.net.HttpURLConnection;
24+
import java.net.URI;
2325
import java.net.URL;
2426
import java.nio.file.Path;
2527
import java.time.Duration;
28+
import java.util.UUID;
2629
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
2730
import org.junit.jupiter.api.Test;
2831
import org.junit.jupiter.api.io.TempDir;
32+
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
33+
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
34+
import software.amazon.awssdk.core.ResponseBytes;
35+
import software.amazon.awssdk.core.sync.RequestBody;
36+
import software.amazon.awssdk.regions.Region;
37+
import software.amazon.awssdk.services.s3.S3Client;
38+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
39+
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
2940

3041
/**
3142
* Integration tests for the S3 Gateway of the {@code ozone local} runtime.
@@ -50,6 +61,46 @@ void s3GatewayServesRequests() throws Exception {
5061
}
5162
}
5263

64+
@Test
65+
void awsSdkCanCreateListPutAndGetAgainstLocalRuntime() throws Exception {
66+
// Non-default credentials prove the launcher provisions whatever key pair is configured.
67+
LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
68+
tempDir.resolve("local-ozone-s3-sdk"))
69+
.setS3AccessKey("localuser")
70+
.setS3SecretKey("localsecret")
71+
.setStartupTimeout(Duration.ofMinutes(3))
72+
.build();
73+
74+
String bucketName = "local-" + UUID.randomUUID().toString().replace("-", "");
75+
String keyName = "key-" + UUID.randomUUID().toString().replace("-", "");
76+
String payload = "local-ozone-s3";
77+
78+
try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, new OzoneConfiguration())) {
79+
cluster.start();
80+
81+
try (S3Client client = S3Client.builder()
82+
.region(Region.of(config.getS3Region()))
83+
.endpointOverride(URI.create(cluster.getS3Endpoint()))
84+
.credentialsProvider(StaticCredentialsProvider.create(
85+
AwsBasicCredentials.create(config.getS3AccessKey(), config.getS3SecretKey())))
86+
.forcePathStyle(true)
87+
.build()) {
88+
client.createBucket(builder -> builder.bucket(bucketName));
89+
90+
ListBucketsResponse buckets = client.listBuckets();
91+
assertTrue(buckets.buckets().stream()
92+
.anyMatch(bucket -> bucketName.equals(bucket.name())));
93+
94+
client.putObject(builder -> builder.bucket(bucketName).key(keyName),
95+
RequestBody.fromString(payload));
96+
97+
ResponseBytes<GetObjectResponse> response = client.getObjectAsBytes(
98+
builder -> builder.bucket(bucketName).key(keyName));
99+
assertEquals(payload, response.asUtf8String());
100+
}
101+
}
102+
}
103+
53104
private static void assertS3EndpointResponds(String endpoint) throws Exception {
54105
HttpURLConnection connection = (HttpURLConnection) new URL(endpoint).openConnection();
55106
try {

0 commit comments

Comments
 (0)