diff --git a/dev/test.http b/dev/test.http index 64557f5..8c19f55 100644 --- a/dev/test.http +++ b/dev/test.http @@ -1,22 +1,34 @@ -### upload a payload -POST localhost:8080/upload -Content-Type: multipart/form-data; boundary=Boundary -Authorization: Basic admin admin - ---Boundary -Content-Disposition: form-data; name="request" +### create a presigned upload URL +POST localhost:8080/v3/upload Content-Type: application/json +Authorization: Basic admin admin { - "id": "6e9e34e8-b8ec-4c11-9b23-16dd951bb480" + "id": "6e9e34e8-b8ec-4c11-9b23-16dd951bb480", + "download": { + "name": "test.txt", + "checksums": { + "sha256": "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" + }, + "size": 6 + }, + "contentType": "application/java-archive", + "contentMd5": "CffgLxKQviEdpweiZvFTsw==" } ---Boundary -Content-Disposition: form-data; name="file"; filename="test.txt" + +> {% client.global.set("uploadUrl", response.body.url); %} + +### upload the payload +PUT {{uploadUrl}} +Content-Length: 6 +Content-MD5: CffgLxKQviEdpweiZvFTsw== +Content-Type: application/java-archive +x-amz-meta-sha256: 66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18 < test.txt ### publish a build -POST localhost:8080/publish +POST localhost:8080/v3/publish Content-Type: application/json Authorization: Basic admin admin @@ -39,7 +51,7 @@ Authorization: Basic admin admin "checksums": { "sha256": "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18" }, - "size": 1 + "size": 6 } } } diff --git a/src/main/java/io/papermc/fill/controller/Publish3Controller.java b/src/main/java/io/papermc/fill/controller/Publish3Controller.java new file mode 100644 index 0000000..09471c9 --- /dev/null +++ b/src/main/java/io/papermc/fill/controller/Publish3Controller.java @@ -0,0 +1,219 @@ +/* + * Copyright 2024 PaperMC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.papermc.fill.controller; + +import io.papermc.fill.database.BuildEntity; +import io.papermc.fill.database.BuildRepository; +import io.papermc.fill.database.FamilyEntity; +import io.papermc.fill.database.FamilyRepository; +import io.papermc.fill.database.ProjectEntity; +import io.papermc.fill.database.ProjectRepository; +import io.papermc.fill.database.VersionEntity; +import io.papermc.fill.database.VersionRepository; +import io.papermc.fill.exception.DuplicateBuildException; +import io.papermc.fill.exception.FamilyNotFoundException; +import io.papermc.fill.exception.ProjectNotFoundException; +import io.papermc.fill.exception.PublishFailedException; +import io.papermc.fill.exception.StorageWriteException; +import io.papermc.fill.exception.VersionNotFoundException; +import io.papermc.fill.model.Commit; +import io.papermc.fill.model.Download; +import io.papermc.fill.model.Support; +import io.papermc.fill.model.request.PublishRequest; +import io.papermc.fill.model.request.v3.UploadRequest; +import io.papermc.fill.model.response.PublishResponse; +import io.papermc.fill.model.response.v3.UploadResponse; +import io.papermc.fill.notification.BuildListener; +import io.papermc.fill.service.StorageService; +import io.papermc.fill.util.http.Responses; +import java.time.Instant; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.bson.types.ObjectId; +import org.jspecify.annotations.NullMarked; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.CrossOrigin; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +@NullMarked +@RestController +public class Publish3Controller { + private static final boolean CREATE_MISSING_VERSIONS = true; + private static final Logger LOGGER = LoggerFactory.getLogger(Publish3Controller.class); + + private final ProjectRepository projects; + private final FamilyRepository families; + private final VersionRepository versions; + private final BuildRepository builds; + private final StorageService storage; + private final Set buildListeners; + + @Autowired + public Publish3Controller( + final ProjectRepository projects, + final FamilyRepository families, + final VersionRepository versions, + final BuildRepository builds, + final StorageService storage, + final Set buildListeners + ) { + this.projects = projects; + this.families = families; + this.versions = versions; + this.builds = builds; + this.storage = storage; + this.buildListeners = buildListeners; + } + + @CrossOrigin(methods = RequestMethod.POST) + @PostMapping( + consumes = MediaType.APPLICATION_JSON_VALUE, + path = "/v3/upload" + ) + @PreAuthorize("hasRole('API_PUBLISH')") + public ResponseEntity upload(@RequestBody final UploadRequest request) { + if (request.download().name().isBlank() || request.download().checksums().sha256().isBlank() || request.download().size() < 0 || request.contentType().isBlank() || request.contentMd5().isBlank()) { + final String message = "Invalid upload metadata"; + throw createPublishFailedException(request, message, new IllegalArgumentException(message)); + } + try { + return Responses.ok(new UploadResponse( + true, + this.storage.createUploadUrl(request.id(), request.download(), request.contentMd5(), MediaType.parseMediaType(request.contentType())) + )); + } catch (final StorageWriteException | IllegalArgumentException e) { + throw createPublishFailedException(request, "Could not create upload URL", e); + } + } + + @CrossOrigin(methods = RequestMethod.POST) + @PostMapping( + consumes = MediaType.APPLICATION_JSON_VALUE, + path = "/v3/publish" + ) + @PreAuthorize("hasRole('API_PUBLISH')") + public ResponseEntity publish( + @RequestBody + final PublishRequest request + ) { + final Instant createdAt = request.time(); + + final ProjectEntity project = this.projects.findByKey(request.project()).orElseThrow(ProjectNotFoundException::new); + final FamilyEntity family = this.families.findByProjectAndKey(project, request.family()).orElseThrow(FamilyNotFoundException::new); + VersionEntity version = this.versions.findByProjectAndKey(project, request.version()).orElse(null); + if (version == null) { + if (CREATE_MISSING_VERSIONS) { + version = this.versions.save(VersionEntity.create( + new ObjectId(Date.from(createdAt)), + createdAt, + project, + family, + request.version(), + null, + Support.SUPPORTED, + null + )); + } else { + throw new VersionNotFoundException(); + } + } + + final List commits = request.commits().reversed(); + final Map downloads = request.downloads(); + + final BuildEntity existingBuild = this.builds.findByVersionAndNumber(version, request.build()).orElse(null); + if (existingBuild != null) { + if (isSameBuild(existingBuild, request, commits, downloads)) { + this.deleteStagedObjects(request, downloads); + return Responses.created(new PublishResponse(true, existingBuild._id())); + } + throw createPublishFailedException(request, "Build already exists", new DuplicateBuildException()); + } + + final BuildEntity build = BuildEntity.create( + new ObjectId(Date.from(createdAt)), + createdAt, + project, + version, + request.build(), + request.channel(), + commits, + downloads + ); + + for (final Download download : downloads.values()) { + try { + this.storage.verifyStagedObject(request.id(), download); + } catch (final StorageWriteException e) { + throw createPublishFailedException(request, String.format("Could not verify staged object for %s", download.name()), e); + } + } + + for (final Download download : downloads.values()) { + try { + this.storage.promoteStagedObject(request.id(), project, version, build, download); + } catch (final StorageWriteException e) { + throw createPublishFailedException(request, String.format("Could not promote staged object for %s", download.name()), e); + } + } + + this.builds.save(build); + this.deleteStagedObjects(request, downloads); + + for (final BuildListener listener : this.buildListeners) { + listener.onBuildPublished(project, version, build); + } + + return Responses.created(new PublishResponse(true, build._id())); + } + + private static PublishFailedException createPublishFailedException(final Object request, final String message, final Throwable throwable) { + LOGGER.error("Failed to publish [{}]: {}", request, message, throwable); + return new PublishFailedException("Publishing the build failed: " + message, throwable); + } + + private static boolean isSameBuild( + final BuildEntity build, + final PublishRequest request, + final List commits, + final Map downloads + ) { + return build.createdAt().equals(request.time()) && + build.channel() == request.channel() && + build.commits().equals(commits) && + build.downloads().equals(downloads); + } + + private void deleteStagedObjects(final PublishRequest request, final Map downloads) { + for (final Download download : downloads.values()) { + try { + this.storage.deleteStagedObject(request.id(), download.name()); + } catch (final StorageWriteException e) { + LOGGER.warn("Failed to delete staged object for [{}]", download.name(), e); + } + } + } +} diff --git a/src/main/java/io/papermc/fill/model/request/v3/UploadRequest.java b/src/main/java/io/papermc/fill/model/request/v3/UploadRequest.java new file mode 100644 index 0000000..48136ab --- /dev/null +++ b/src/main/java/io/papermc/fill/model/request/v3/UploadRequest.java @@ -0,0 +1,29 @@ +/* + * Copyright 2024 PaperMC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.papermc.fill.model.request.v3; + +import io.papermc.fill.model.Download; +import java.util.UUID; +import org.jspecify.annotations.NullMarked; + +@NullMarked +public record UploadRequest( + UUID id, + Download download, + String contentType, + String contentMd5 +) { +} diff --git a/src/main/java/io/papermc/fill/model/response/v3/UploadResponse.java b/src/main/java/io/papermc/fill/model/response/v3/UploadResponse.java new file mode 100644 index 0000000..0a5e9b7 --- /dev/null +++ b/src/main/java/io/papermc/fill/model/response/v3/UploadResponse.java @@ -0,0 +1,26 @@ +/* + * Copyright 2024 PaperMC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.papermc.fill.model.response.v3; + +import java.net.URI; +import org.jspecify.annotations.NullMarked; + +@NullMarked +public record UploadResponse( + boolean ok, + URI url +) { +} diff --git a/src/main/java/io/papermc/fill/s3/S3Configuration.java b/src/main/java/io/papermc/fill/s3/S3Configuration.java index 18d40d1..80a2a67 100644 --- a/src/main/java/io/papermc/fill/s3/S3Configuration.java +++ b/src/main/java/io/papermc/fill/s3/S3Configuration.java @@ -25,6 +25,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; @NullMarked public interface S3Configuration { @@ -62,4 +63,19 @@ static S3Client createClient(final S3Configuration properties) { }); return client.build(); } + + static S3Presigner createPresigner(final S3Configuration properties) { + final S3Presigner.Builder presigner = S3Presigner.builder() + .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(properties.accessKeyId(), properties.secretAccessKey()))) + .region(Region.of(properties.region())) + .serviceConfiguration(software.amazon.awssdk.services.s3.S3Configuration.builder() + .pathStyleAccessEnabled(properties.usePathStyleAccess()) + .multiRegionEnabled(false) + .build()); + final URI endpoint = properties.endpoint(); + if (endpoint != null) { + presigner.endpointOverride(endpoint); + } + return presigner.build(); + } } diff --git a/src/main/java/io/papermc/fill/service/StorageService.java b/src/main/java/io/papermc/fill/service/StorageService.java index 45070a9..d97578a 100644 --- a/src/main/java/io/papermc/fill/service/StorageService.java +++ b/src/main/java/io/papermc/fill/service/StorageService.java @@ -24,6 +24,7 @@ import io.papermc.fill.model.Version; import java.net.URI; import java.util.Map; +import java.util.UUID; import org.apache.commons.text.StringSubstitutor; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -83,6 +84,31 @@ void putObject( final MimeType type ) throws StorageWriteException; + URI createUploadUrl( + final UUID id, + final Download download, + final String contentMd5, + final MimeType type + ) throws StorageWriteException; + + void verifyStagedObject( + final UUID id, + final Download download + ) throws StorageWriteException; + + void promoteStagedObject( + final UUID id, + final Project project, + final Version version, + final BuildWithDownloads build, + final Download download + ) throws StorageWriteException; + + void deleteStagedObject( + final UUID id, + final String filename + ) throws StorageWriteException; + @Deprecated @Nullable Asset getObject( final Project project, diff --git a/src/main/java/io/papermc/fill/service/StorageServiceImpl.java b/src/main/java/io/papermc/fill/service/StorageServiceImpl.java index 86becee..2aeaf9b 100644 --- a/src/main/java/io/papermc/fill/service/StorageServiceImpl.java +++ b/src/main/java/io/papermc/fill/service/StorageServiceImpl.java @@ -24,9 +24,13 @@ import io.papermc.fill.model.Version; import io.papermc.fill.s3.S3Configuration; import io.papermc.fill.util.http.Headers; +import jakarta.annotation.PreDestroy; import java.io.IOException; import java.net.URI; +import java.time.Duration; +import java.util.Map; import java.util.NoSuchElementException; +import java.util.UUID; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; import org.slf4j.Logger; @@ -42,17 +46,27 @@ import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; @NullMarked @Service public class StorageServiceImpl implements StorageService { + // The bucket must expire abandoned objects under this prefix with a lifecycle rule + // to prevent object leaks from failed publications. + private static final String STAGING_PREFIX = "staging/"; + private static final Duration UPLOAD_URL_DURATION = Duration.ofMinutes(15); private static final Logger LOGGER = LoggerFactory.getLogger(StorageServiceImpl.class); private final ApplicationApiProperties properties; private final S3Client s3; + private final S3Presigner presigner; private final RestClient http; @Autowired @@ -61,6 +75,7 @@ public StorageServiceImpl( ) { this.properties = properties; this.s3 = S3Configuration.createClient(properties.storage().s3()); + this.presigner = S3Configuration.createPresigner(properties.storage().s3()); this.http = RestClient.builder() .defaultHeader(HttpHeaders.USER_AGENT, "Fill (Internal)") .build(); @@ -87,18 +102,101 @@ public void putObject( ) throws StorageWriteException { final ApplicationApiProperties.Storage properties = this.properties.storage(); final String path = StorageService.createPath(properties.path(), project, version, build, download); - final PutObjectRequest.Builder request = PutObjectRequest.builder() + final PutObjectRequest request = PutObjectRequest.builder() .bucket(properties.s3().bucket()) .key(path) .contentLength((long) content.length) - .contentType(type.toString()); + .contentType(type.toString()) + .build(); + try { + this.s3.putObject(request, RequestBody.fromBytes(content)); + } catch (final SdkException e) { + throw createStorageWriteException(download, path, "s3 exception", e); + } + } + + @Override + public URI createUploadUrl( + final UUID id, + final Download download, + final String contentMd5, + final MimeType type + ) throws StorageWriteException { + final String path = stagingPath(id, download.name()); + final PutObjectRequest request = PutObjectRequest.builder() + .bucket(this.properties.storage().s3().bucket()) + .key(path) + .contentLength((long) download.size()) + .contentType(type.toString()) + .contentMD5(contentMd5) + .metadata(Map.of("sha256", download.checksums().sha256())) + .build(); try { - this.s3.putObject(request.build(), RequestBody.fromBytes(content)); + return URI.create(this.presigner.presignPutObject(PutObjectPresignRequest.builder() + .signatureDuration(UPLOAD_URL_DURATION) + .putObjectRequest(request) + .build()).url().toString()); + } catch (final SdkException e) { + throw createStorageWriteException(download, path, "s3 exception", e); + } + } + + @Override + public void verifyStagedObject(final UUID id, final Download download) throws StorageWriteException { + final String path = stagingPath(id, download.name()); + try { + final var response = this.s3.headObject(HeadObjectRequest.builder() + .bucket(this.properties.storage().s3().bucket()) + .key(path) + .build()); + if (response.contentLength() != download.size()) { + throw createStorageWriteException(download, path, String.format("expected size %d but got %d", download.size(), response.contentLength()), new IllegalArgumentException()); + } + final String actualSha256 = response.metadata().get("sha256"); + if (!download.checksums().sha256().equals(actualSha256)) { + throw createStorageWriteException(download, path, String.format("expected SHA-256 %s but got %s", download.checksums().sha256(), actualSha256), new IllegalArgumentException()); + } } catch (final SdkException e) { throw createStorageWriteException(download, path, "s3 exception", e); } } + @Override + public void promoteStagedObject( + final UUID id, + final Project project, + final Version version, + final BuildWithDownloads build, + final Download download + ) throws StorageWriteException { + final ApplicationApiProperties.Storage properties = this.properties.storage(); + final String source = stagingPath(id, download.name()); + final String destination = StorageService.createPath(properties.path(), project, version, build, download); + try { + this.s3.copyObject(CopyObjectRequest.builder() + .sourceBucket(properties.s3().bucket()) + .sourceKey(source) + .destinationBucket(properties.s3().bucket()) + .destinationKey(destination) + .build()); + } catch (final SdkException e) { + throw createStorageWriteException(download, destination, "s3 exception", e); + } + } + + @Override + public void deleteStagedObject(final UUID id, final String filename) throws StorageWriteException { + final String path = stagingPath(id, filename); + try { + this.s3.deleteObject(DeleteObjectRequest.builder() + .bucket(this.properties.storage().s3().bucket()) + .key(path) + .build()); + } catch (final SdkException e) { + throw createStorageWriteException(filename, path, "s3 exception", e); + } + } + @Deprecated @Override public @Nullable Asset getObject( @@ -152,14 +250,24 @@ public void putObject( }; } + private static String stagingPath(final UUID id, final String filename) { + return String.format("%s%s/%s", STAGING_PREFIX, id, filename); + } + + @PreDestroy + public void close() { + this.presigner.close(); + this.s3.close(); + } + private static StorageReadException createStorageReadException(final Download download, final Object path, final String reason, final Throwable throwable) { final String message = String.format("Failed to read object [%s] from storage [%s]: %s", download, path, reason); LOGGER.error(message, throwable); return new StorageReadException(message, throwable); } - private static StorageWriteException createStorageWriteException(final Download download, final Object path, final String reason, final Throwable throwable) { - final String message = String.format("Failed to write object [%s] to storage [%s]: %s", download, path, reason); + private static StorageWriteException createStorageWriteException(final Object object, final Object path, final String reason, final Throwable throwable) { + final String message = String.format("Failed to write object [%s] to storage [%s]: %s", object, path, reason); LOGGER.error(message, throwable); return new StorageWriteException(message, throwable); } diff --git a/src/test/java/io/papermc/fill/controller/Publish3ControllerTest.java b/src/test/java/io/papermc/fill/controller/Publish3ControllerTest.java new file mode 100644 index 0000000..1075930 --- /dev/null +++ b/src/test/java/io/papermc/fill/controller/Publish3ControllerTest.java @@ -0,0 +1,213 @@ +/* + * Copyright 2024 PaperMC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.papermc.fill.controller; + +import io.papermc.fill.database.BuildEntity; +import io.papermc.fill.database.BuildRepository; +import io.papermc.fill.database.FamilyEntity; +import io.papermc.fill.database.FamilyRepository; +import io.papermc.fill.database.ProjectEntity; +import io.papermc.fill.database.ProjectRepository; +import io.papermc.fill.database.VersionEntity; +import io.papermc.fill.database.VersionRepository; +import io.papermc.fill.exception.PublishFailedException; +import io.papermc.fill.exception.StorageWriteException; +import io.papermc.fill.model.BuildChannel; +import io.papermc.fill.model.Checksums; +import io.papermc.fill.model.Commit; +import io.papermc.fill.model.Download; +import io.papermc.fill.model.Java; +import io.papermc.fill.model.JavaFlags; +import io.papermc.fill.model.JavaVersion; +import io.papermc.fill.model.Support; +import io.papermc.fill.model.request.PublishRequest; +import io.papermc.fill.notification.BuildListener; +import io.papermc.fill.service.StorageService; +import io.papermc.fill.util.discord.DiscordNotificationChannel; +import io.papermc.fill.util.git.GitRepository; +import java.net.URI; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import org.bson.types.ObjectId; +import org.jspecify.annotations.NullMarked; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.springframework.http.HttpStatus; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +@NullMarked +public class Publish3ControllerTest { + private static final Instant CREATED_AT = Instant.parse("2026-07-28T00:00:00Z"); + private static final UUID UPLOAD_ID = UUID.fromString("9d42dfd6-6b0f-4eb5-ac5f-45efcdfead7e"); + private static final ProjectEntity PROJECT = ProjectEntity.create( + new ObjectId("000000000000000000000001"), + "paper", + "Paper", + new GitRepository("PaperMC", "Paper"), + URI.create("https://example.invalid/logo.png"), + List.of(), + "server:default" + ); + private static final FamilyEntity FAMILY = FamilyEntity.create( + new ObjectId("000000000000000000000002"), + CREATED_AT, + PROJECT, + "1.21", + new Java(new JavaVersion(21), new JavaFlags(List.of())) + ); + private static final VersionEntity VERSION = VersionEntity.create( + new ObjectId("000000000000000000000003"), + CREATED_AT, + PROJECT, + FAMILY, + "1.21.8", + null, + Support.SUPPORTED, + null + ); + + private ProjectRepository projects; + private FamilyRepository families; + private VersionRepository versions; + private BuildRepository builds; + private StorageService storage; + private BuildListener listener; + private Publish3Controller controller; + + @BeforeEach + void setup() { + this.projects = mock(ProjectRepository.class); + this.families = mock(FamilyRepository.class); + this.versions = mock(VersionRepository.class); + this.builds = mock(BuildRepository.class); + this.storage = mock(StorageService.class); + this.listener = mock(BuildListener.class); + this.controller = new Publish3Controller( + this.projects, + this.families, + this.versions, + this.builds, + this.storage, + Set.of(this.listener) + ); + + when(this.projects.findByKey(PROJECT.key())).thenReturn(Optional.of(PROJECT)); + when(this.families.findByProjectAndKey(PROJECT, FAMILY.key())).thenReturn(Optional.of(FAMILY)); + when(this.versions.findByProjectAndKey(PROJECT, VERSION.key())).thenReturn(Optional.of(VERSION)); + } + + @Test + void publishesOnlyAfterAllObjectsAreVerifiedAndPromoted() throws Exception { + final PublishRequest request = request(); + final List downloads = List.copyOf(request.downloads().values()); + when(this.builds.findByVersionAndNumber(VERSION, request.build())).thenReturn(Optional.empty()); + when(this.builds.save(any(BuildEntity.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + final var response = this.controller.publish(request); + + assertEquals(HttpStatus.CREATED, response.getStatusCode()); + final InOrder order = inOrder(this.storage, this.builds, this.listener); + for (final Download download : downloads) { + order.verify(this.storage).verifyStagedObject(UPLOAD_ID, download); + } + for (final Download download : downloads) { + order.verify(this.storage).promoteStagedObject(eq(UPLOAD_ID), eq(PROJECT), eq(VERSION), any(BuildEntity.class), eq(download)); + } + order.verify(this.builds).save(any(BuildEntity.class)); + for (final Download download : downloads) { + order.verify(this.storage).deleteStagedObject(UPLOAD_ID, download.name()); + } + order.verify(this.listener).onBuildPublished(eq(PROJECT), eq(VERSION), any(BuildEntity.class)); + } + + @Test + void doesNotPublishOrCleanUpWhenPromotionFails() throws Exception { + final PublishRequest request = request(); + final List downloads = List.copyOf(request.downloads().values()); + when(this.builds.findByVersionAndNumber(VERSION, request.build())).thenReturn(Optional.empty()); + doThrow(new StorageWriteException("copy failed", new IllegalStateException())) + .when(this.storage).promoteStagedObject(eq(UPLOAD_ID), eq(PROJECT), eq(VERSION), any(BuildEntity.class), eq(downloads.getFirst())); + + assertThrows(PublishFailedException.class, () -> this.controller.publish(request)); + + verify(this.builds, never()).save(any(BuildEntity.class)); + for (final Download download : downloads) { + verify(this.storage, never()).deleteStagedObject(UPLOAD_ID, download.name()); + } + verifyNoInteractions(this.listener); + } + + @Test + void treatsAnIdenticalExistingBuildAsAnIdempotentRetry() throws Exception { + final PublishRequest request = request(); + final BuildEntity existing = BuildEntity.create( + new ObjectId("000000000000000000000004"), + request.time(), + PROJECT, + VERSION, + request.build(), + request.channel(), + request.commits().reversed(), + request.downloads() + ); + when(this.builds.findByVersionAndNumber(VERSION, request.build())).thenReturn(Optional.of(existing)); + + final var response = this.controller.publish(request); + + assertEquals(HttpStatus.CREATED, response.getStatusCode()); + for (final Download download : request.downloads().values()) { + verify(this.storage).deleteStagedObject(UPLOAD_ID, download.name()); + } + verifyNoMoreInteractions(this.storage); + verify(this.builds, never()).save(any(BuildEntity.class)); + verifyNoInteractions(this.listener); + } + + private static PublishRequest request() { + final Map downloads = new LinkedHashMap<>(); + downloads.put("server:default", new Download("paper.jar", new Checksums("a".repeat(64)), 100)); + downloads.put("server:mojang", new Download("paper-mojang.jar", new Checksums("b".repeat(64)), 200)); + return new PublishRequest( + UPLOAD_ID, + PROJECT.key(), + FAMILY.key(), + VERSION.key(), + 42, + CREATED_AT, + BuildChannel.STABLE, + List.of(new Commit("c".repeat(40), CREATED_AT, "Test commit")), + downloads + ); + } +} diff --git a/src/test/java/io/papermc/fill/s3/S3ConfigurationTest.java b/src/test/java/io/papermc/fill/s3/S3ConfigurationTest.java new file mode 100644 index 0000000..3b9eb80 --- /dev/null +++ b/src/test/java/io/papermc/fill/s3/S3ConfigurationTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2024 PaperMC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.papermc.fill.s3; + +import java.net.URI; +import java.time.Duration; +import java.util.Map; +import org.jspecify.annotations.NullMarked; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@NullMarked +public class S3ConfigurationTest { + @Test + void createsSigV4PresignedUploadUrl() { + try (final var presigner = S3Configuration.createPresigner(new TestConfiguration())) { + final var request = presigner.presignPutObject(PutObjectPresignRequest.builder() + .signatureDuration(Duration.ofMinutes(5)) + .putObjectRequest(PutObjectRequest.builder() + .bucket("fill") + .key("staging/test/file.jar") + .contentLength(10L) + .contentMD5("6Afx/PgtEy+bsBjKZzihnw==") + .contentType("application/java-archive") + .metadata(Map.of("sha256", "test-sha256")) + .build()) + .build()); + + assertTrue(request.url().getQuery().contains("X-Amz-Algorithm=AWS4-HMAC-SHA256")); + assertEquals("10", request.signedHeaders().get("content-length").getFirst()); + assertEquals("6Afx/PgtEy+bsBjKZzihnw==", request.signedHeaders().get("content-md5").getFirst()); + assertEquals("application/java-archive", request.signedHeaders().get("content-type").getFirst()); + assertEquals("test-sha256", request.signedHeaders().get("x-amz-meta-sha256").getFirst()); + } + } + + private record TestConfiguration() implements S3Configuration { + @Override + public URI endpoint() { + return URI.create("https://example.invalid"); + } + + @Override + public String region() { + return "auto"; + } + + @Override + public String accessKeyId() { + return "access-key"; + } + + @Override + public String secretAccessKey() { + return "secret-key"; + } + + @Override + public String bucket() { + return "fill"; + } + + @Override + public boolean usePathStyleAccess() { + return true; + } + + @Override + public boolean useS3v4Signer() { + return false; + } + } +}