2025/04 API and DTO sync

CHANGELOG.md

@@ -123,6 +123,8 @@ Running S3Mock in unit tests is still supported by using [TestContainers](https:
 * Refactorings
   * AWS has deprecated SDK for Java v1, and will remove support EOY 2025.
     * S3Mock will remove usage of Java v1 early 2026.
+  * JUnit 4.x deprecation
+    * JUnit 4.x will be removed from the code base.
   * Looking to Remove unit test modules. This enables
     * Refactoring S3Mock to a "standard" Spring Boot application.
     * Removal of workarounds to use `S3MockApplication#start` from a static context
@@ -144,12 +146,33 @@ Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
 * Features and fixes
   * Support checksum algorithm CRC64NVME (fixes #2334)
 * Refactorings
-  * TBD
+  * API / DTO consistency check 2025/04
+    * Check AWS API for changes
+    * Update S3Mock API / DTOs
+    * Add tests for changed API / DTOs
+    * CreateBucket API now accepts "CreateBucketConfiguration" request body
+    * HeadBucket API now returns region and location headers
+    * CompleteMultipartUpload API now accepts checksums and returns checksums
+    * ListObjects API now returns "delimiter"
+    * ListObjects V2 API now accepts "fetch-owner" and returns "delimiter"
+    * ListBuckets API now accepts parameters listed in AWS S3 API
+    * ListMultipartUploads now accepts parameters listed in AWS S3 API
+    * ListParts now accepts parameters listed in AWS S3 API
+    * UploadPartCopy now accepts and returns encryption headers
+    * CreateMultipartUpload now accepts checksum headers and returns checksum and encryption headers
+    * CompleteMultipartUpload now accepts checksum headers and returns checksum and encryption headers
+      * Checksum validation on complete
+    * DeleteObject now supports conditional requests
+    * PutObject now supports conditional requests
 * Version updates (deliverable dependencies)
+  * Bump aws-v2.version from 2.31.25 to 2.31.37
+  * Bump aws.sdk.kotlin:s3-jvm from 1.4.67 to 1.4.79
+  * Bump aws.version from 1.12.782 to 1.12.783
   * Bump spring-boot.version from 3.4.4 to 3.4.5
   * Bump testcontainers.version from 1.20.6 to 1.21.0
 * Version updates (build dependencies)
-  * Bump github/codeql-action from 3.28.15 to 3.28.16
+  * Bump github/codeql-action from 3.28.15 to 3.28.17
+  * Bump com.puppycrawl.tools:checkstyle from 10.23.0 to 10.23.1
 
 ## 4.1.1
 Version 4.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.

build-config/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.1.2-SNAPSHOT</version>
+    <version>4.2.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock-build-config</artifactId>

build-config/src/main/resources/build-config/checkstyle.xml

@@ -42,7 +42,7 @@
   </module>
 
   <module name="LineLength">
-    <property name="max" value="100"/>
+    <property name="max" value="120"/>
     <property name="fileExtensions" value="java"/>
     <property name="ignorePattern"
       value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>

docker/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.1.2-SNAPSHOT</version>
+    <version>4.2.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock-docker</artifactId>

integration-tests/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>com.adobe.testing</groupId>
     <artifactId>s3mock-parent</artifactId>
-    <version>4.1.2-SNAPSHOT</version>
+    <version>4.2.0-SNAPSHOT</version>
   </parent>
 
   <artifactId>s3mock-integration-tests</artifactId>

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/AwsChunkedEncodingIT.kt

@@ -23,6 +23,7 @@ import org.junit.jupiter.api.TestInfo
 import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm
 import software.amazon.awssdk.core.sync.RequestBody
 import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
+import software.amazon.awssdk.services.s3.model.ChecksumMode
 import java.io.File
 import java.io.FileInputStream
 import java.io.InputStream
@@ -67,6 +68,7 @@ internal class AwsChunkedEncodingIT : S3TestBase() {
 
     s3Client.getObject {
       it.bucket(bucket)
+      it.checksumMode(ChecksumMode.ENABLED)
       it.key(UPLOAD_FILE_NAME)
     }.also {
       assertThat(it.response().eTag()).isEqualTo(expectedEtag)

integration-tests/src/test/kotlin/com/adobe/testing/s3mock/its/BucketIT.kt

@@ -26,13 +26,16 @@ import software.amazon.awssdk.awscore.exception.AwsServiceException
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.AbortIncompleteMultipartUpload
 import software.amazon.awssdk.services.s3.model.BucketLifecycleConfiguration
+import software.amazon.awssdk.services.s3.model.BucketType
 import software.amazon.awssdk.services.s3.model.BucketVersioningStatus
+import software.amazon.awssdk.services.s3.model.DataRedundancy
 import software.amazon.awssdk.services.s3.model.DeleteBucketRequest
 import software.amazon.awssdk.services.s3.model.ExpirationStatus
 import software.amazon.awssdk.services.s3.model.GetBucketLifecycleConfigurationRequest
 import software.amazon.awssdk.services.s3.model.LifecycleExpiration
 import software.amazon.awssdk.services.s3.model.LifecycleRule
 import software.amazon.awssdk.services.s3.model.LifecycleRuleFilter
+import software.amazon.awssdk.services.s3.model.LocationType
 import software.amazon.awssdk.services.s3.model.MFADelete
 import software.amazon.awssdk.services.s3.model.MFADeleteStatus
 import software.amazon.awssdk.services.s3.model.NoSuchBucketException
@@ -68,6 +71,40 @@ internal class BucketIT : S3TestBase() {
     }
   }
 
+  /**
+   * Not sure why S3 does not accept any combination of LocationInfo and BucketInfo in the configuration.
+   * Requests always fail claiming that the XML is not well-formed, even though it is generated by their own SDK...
+   */
+  @Test
+  @S3VerifiedFailure(year = 2025,
+    reason = "The XML you provided was not well-formed or did not validate against our published schema")
+  fun `creating a bucket with configuration is successful`(testInfo: TestInfo) {
+    val bucketName = bucketName(testInfo)
+    val createBucketResponse = s3Client.createBucket {
+      it.bucket(bucketName)
+      it.createBucketConfiguration {
+        it.locationConstraint("ap-southeast-5")
+        it.bucket {
+          it.dataRedundancy(DataRedundancy.SINGLE_AVAILABILITY_ZONE)
+          it.type(BucketType.DIRECTORY)
+        }
+        it.location {
+          it.name("SomeName")
+          it.type(LocationType.AVAILABILITY_ZONE)
+        }
+      }
+    }
+    assertThat(createBucketResponse.sdkHttpResponse().statusCode()).isEqualTo(200)
+    assertThat(createBucketResponse.location()).isEqualTo("/$bucketName")
+
+    val bucketCreated = s3Client.waiter().waitUntilBucketExists { it.bucket(bucketName) }
+    val bucketCreatedResponse = bucketCreated.matched().response().get()
+    assertThat(bucketCreatedResponse).isNotNull
+
+    //does not throw exception if bucket exists.
+    s3Client.headBucket { it.bucket(bucketName) }
+  }
+
   @Test
   @S3VerifiedSuccess(year = 2025)
   fun `deleting a non-empty bucket fails`(testInfo: TestInfo) {
@@ -94,20 +131,113 @@ internal class BucketIT : S3TestBase() {
     // and account for a clock-skew in the Docker container of up to a minute.
     val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES)
 
-    s3Client.listBuckets{
+    s3Client.listBuckets { }.also {
+      assertThat(it.hasBuckets()).isTrue
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(5)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          // the default buckets
+          "bucket-a",
+          "bucket-b",
+          // the buckets we created in this test
+          "${bucketName}-1",
+          "${bucketName}-2",
+          "${bucketName}-3"
+        )
+        assertThat(it[2].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[3].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[4].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isNull()
+      assertThat(it.continuationToken()).isNull()
+      assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
+      assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }
+  }
+
+  @Test
+  @S3VerifiedFailure(year = 2025,
+    reason = "Default owner does not exist in S3.")
+  fun `creating and listing multiple buckets limiting by prefix is successful`(testInfo: TestInfo) {
+    val bucketName = bucketName(testInfo)
+    givenBucket("${bucketName}-1")
+    givenBucket("${bucketName}-2")
+    givenBucket("${bucketName}-3")
+    // the returned creation date might strip off the millisecond-part, resulting in rounding down
+    // and account for a clock-skew in the Docker container of up to a minute.
+    val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES)
+
+    s3Client.listBuckets {
       it.prefix(bucketName)
     }.also {
       assertThat(it.hasBuckets()).isTrue
-      //TODO: ListBuckets API currently ignores the prefix argument, see #2340
-      it.buckets()
-        .filter { b -> b.name().startsWith(bucketName) }.also { filteredBuckets ->
-          assertThat(filteredBuckets.size).isEqualTo(3)
-          assertThat(filteredBuckets.map { b -> b.name() })
-            .containsExactlyInAnyOrder("${bucketName}-1", "${bucketName}-2", "${bucketName}-3")
-          assertThat(filteredBuckets[0].creationDate()).isAfterOrEqualTo(creationDate)
-          assertThat(filteredBuckets[1].creationDate()).isAfterOrEqualTo(creationDate)
-          assertThat(filteredBuckets[2].creationDate()).isAfterOrEqualTo(creationDate)
-        }
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(3)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          // the buckets we created in this test
+          "${bucketName}-1",
+          "${bucketName}-2",
+          "${bucketName}-3"
+        )
+        assertThat(it[0].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[1].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[2].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isEqualTo(bucketName)
+      assertThat(it.continuationToken()).isNull()
+      assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
+      assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }
+  }
+
+  @Test
+  @S3VerifiedFailure(year = 2025,
+    reason = "Default owner does not exist in S3.")
+  fun `creating and listing multiple buckets limiting by maxBuckets is successful`(testInfo: TestInfo) {
+    val bucketName = bucketName(testInfo)
+    givenBucket("${bucketName}-1")
+    givenBucket("${bucketName}-2")
+    givenBucket("${bucketName}-3")
+    // the returned creation date might strip off the millisecond-part, resulting in rounding down
+    // and account for a clock-skew in the Docker container of up to a minute.
+    val creationDate = Instant.now().minus(1, ChronoUnit.MINUTES)
+
+    val continuationToken = s3Client.listBuckets {
+      it.maxBuckets(4)
+    }.also {
+      assertThat(it.hasBuckets()).isTrue
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(4)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          // the default buckets
+          "bucket-a",
+          "bucket-b",
+          // the buckets we created in this test
+          "${bucketName}-1",
+          "${bucketName}-2"
+        )
+        assertThat(it[2].creationDate()).isAfterOrEqualTo(creationDate)
+        assertThat(it[3].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isNull()
+      assertThat(it.continuationToken()).isNotNull
+      assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
+      assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
+    }.continuationToken()
+
+    s3Client.listBuckets {
+      it.continuationToken(continuationToken)
+    }.also {
+      assertThat(it.hasBuckets()).isTrue
+      it.buckets().also {
+        assertThat(it.size).isEqualTo(1)
+        assertThat(it.map { b -> b.name() }).containsExactly(
+          "${bucketName}-3"
+        )
+        assertThat(it[0].creationDate()).isAfterOrEqualTo(creationDate)
+      }
+      assertThat(it.prefix()).isNull()
+      assertThat(it.continuationToken()).isNull()
       assertThat(it.owner().displayName()).isEqualTo("s3-mock-file-store")
       assertThat(it.owner().id()).isEqualTo("79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be")
     }
@@ -121,7 +251,7 @@ internal class BucketIT : S3TestBase() {
       assertThat(it.buckets())
         .hasSize(2)
         .extracting("name")
-        .containsExactlyInAnyOrder(INITIAL_BUCKET_NAMES.first(), INITIAL_BUCKET_NAMES.last())
+        .containsExactly(INITIAL_BUCKET_NAMES.first(), INITIAL_BUCKET_NAMES.last())
     }
   }