From dac557ea43935a8b40f73513c9599427708467d0 Mon Sep 17 00:00:00 2001 From: Scott Wicken <1562170+swicken@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:47:26 -0400 Subject: [PATCH 1/4] feat(storage): share temporary uploads and WebDAV temporary files through S3 Sixth slice of the S3 asset storage work. With FEATURE_FLAG_S3_ASSET_STORAGE on, completed temporary uploads and their custom metadata are stored in S3 with an access and expiry receipt, WebDAV temporary files publish payloads and path records with conditional writes, and BinaryCleanupJob expires both. Mixed-case temporary names are kept, and flag-off WebDAV and temporary uploads match main. --- docs/testing/BINARY_S3_STORAGE.md | 37 +- .../dotcms/rest/api/v1/temp/TempFileAPI.java | 87 ++++- .../dotcms/storage/FileMetadataAPIImpl.java | 52 ++- .../FileSystemStoragePersistenceAPIImpl.java | 3 +- .../dotcms/storage/TemporaryAssetStorage.java | 254 +++++++++++++ .../storage/WebdavTemporaryStorage.java | 341 ++++++++++++++++++ .../quartz/job/BinaryCleanupJob.java | 12 + .../servlets/BinaryExporterServlet.java | 3 + .../webdav/BasicFolderResourceImpl.java | 11 +- .../dotmarketing/webdav/DotWebdavHelper.java | 202 ++++++++++- .../dotmarketing/webdav/FileResourceImpl.java | 15 + .../webdav/FolderResourceImpl.java | 12 +- .../dotmarketing/webdav/HostResourceImpl.java | 16 +- .../webdav/LanguageFolderResourceImpl.java | 24 +- .../webdav/ResourceFactoryImpl.java | 11 +- .../webdav/TempFileResourceImpl.java | 65 +++- .../webdav/TempFolderResourceImpl.java | 71 +++- .../storage/AssetStorageFeatureTest.java | 26 ++ .../dotcms/storage/BinaryS3StorageTest.java | 274 ++++++++++++++ .../storage/TemporaryAssetStorageTest.java | 143 ++++++++ .../storage/TemporaryMetadataStorageTest.java | 92 +++++ .../webdav/WebdavAssetStorageTest.java | 184 ++++++++++ .../src/test/java/com/dotcms/MainSuite2a.java | 4 +- .../image/focalpoint/FocalPointAPITest.java | 9 +- .../webdav/DotWebdavHelperTest.java | 229 ++++++++++++ 25 files changed, 2119 insertions(+), 58 deletions(-) create mode 100644 dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java create mode 100644 dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java create mode 100644 dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java create mode 100644 dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java create mode 100644 dotCMS/src/test/java/com/dotmarketing/webdav/WebdavAssetStorageTest.java diff --git a/docs/testing/BINARY_S3_STORAGE.md b/docs/testing/BINARY_S3_STORAGE.md index 719edc04c44b..c326c543e530 100644 --- a/docs/testing/BINARY_S3_STORAGE.md +++ b/docs/testing/BINARY_S3_STORAGE.md @@ -80,7 +80,8 @@ The storage chain (`ChainableStoragePersistenceAPI`) and its providers change as completed renditions through the storage chain. Binary assets use the `binary-assets` group under the asset root, laid out as `{inode[0]}/{inode[1]}/{inode}/{field}/{fileName}`. Renditions use the `generated-assets` group under the `dotGenerated` root. With the flag on, keys in these groups keep -their mixed-case names, as do publishing bundles (below); other groups are still lowercased. +their mixed-case names, as do publishing bundles and temporary uploads (below); other groups are +still lowercased. With the flag on, S3 keeps one immutable copy of each set of bytes for these two groups, at `asset-blobs/sha256/////` @@ -327,6 +328,36 @@ that is already gone succeeds. With the flag off, bundles are written, read and deleted in the bundle directory as before. +## Temporary uploads + +With the flag on, completed temporary uploads (`TempFileAPI`, including the image editor's output) +are stored in the `temporary-assets` group, with an immutable receipt that records who may use the +upload and when it expires. Access and expiry are checked before any bytes are downloaded, so +another node can serve the upload after restoring it into its own local root. Mixed-case and +nested names are kept. A payload is published only after the upload stream has closed; an +interrupted upload keeps any existing complete file and never exposes a partial one. Managed +uploads without a receipt cannot fall back to legacy local access. + +Custom metadata for a temporary upload, such as a focal point set before check-in, is stored with +the upload in S3 and read directly from S3, so an edit made on another node is visible. The +scheduled `BinaryCleanupJob` also removes expired S3 temporary uploads, and a failure is logged and +retried on the next run. + +## WebDAV temporary files + +With the flag on, WebDAV temporary files (the scratch files some clients write before the final +upload) keep completed payloads and discoverable path records in the `webdav-temporary` group, so +another node can list and read them. A writer reserves a unique payload, uploads it, and then +publishes the path record with an S3 conditional write, so a stale writer cannot publish after a +deletion. Reads materialize immutable local cache files, so an earlier request keeps its bytes +through a later replacement. Mixed-case temporary names are kept, while normal CMS URLs keep their +case-insensitive handling. + +Cleanup uses S3 server timestamps, protects completed and pending payload references, and +conditionally replaces expired path records with tombstones. Normal WebDAV uploads still go through +content check-in, and their reads, copies and overwrites hold the cache lease through the stream. +With the flag off, WebDAV and temporary uploads use the local filesystem as before. + ## Local cache eviction With the flag on, the local asset directory is a cache that `BinaryCacheEvictionJob` can trim. @@ -425,7 +456,7 @@ docker run -d --rm --name binary-cleanup-postgres-test \ -e POSTGRES_DB=binary_storage_test postgres:16-alpine ./mvnw test -pl :dotcms-core -Dmaven.build.cache.enabled=false \ - -Dtest=AssetStorageFeatureTest,AssetStorageFeatureLatchTest,S3StorageConfigurationTest,NoWebIdentityCredentialsProviderChainTest,BinaryS3StorageTest,BinaryAssetReferenceTest,BinaryCacheEvictionJobTest,BinaryFileSystemStorageTest,BinaryAssetStorageAPIImplTest,MetadataLocalCacheTest,BinaryAssetCleanupTransactionTest,BinaryAssetCleanupProcessorTest,ContentletBackupStorageGateTest,BinaryFieldCleanupProcessorTest,AssetJobEventSerializationTest,BinaryAssetBackfillCheckpointTest,ImportStarterWorkflowCleanupTest,BinaryAssetBackfillProcessorTest,BinaryAssetBackfillTest,ExportStarterFailureTest,BundleArchiveStorageTest,FileAssetBundlerTest \ + -Dtest=AssetStorageFeatureTest,AssetStorageFeatureLatchTest,S3StorageConfigurationTest,NoWebIdentityCredentialsProviderChainTest,BinaryS3StorageTest,BinaryAssetReferenceTest,BinaryCacheEvictionJobTest,BinaryFileSystemStorageTest,BinaryAssetStorageAPIImplTest,MetadataLocalCacheTest,BinaryAssetCleanupTransactionTest,BinaryAssetCleanupProcessorTest,ContentletBackupStorageGateTest,BinaryFieldCleanupProcessorTest,AssetJobEventSerializationTest,BinaryAssetBackfillCheckpointTest,ImportStarterWorkflowCleanupTest,BinaryAssetBackfillProcessorTest,BinaryAssetBackfillTest,ExportStarterFailureTest,BundleArchiveStorageTest,FileAssetBundlerTest,TemporaryAssetStorageTest,WebdavAssetStorageTest,TemporaryMetadataStorageTest \ -Ds3.test.endpoint=http://127.0.0.1:19002 \ -Ds3.test.jdbc=jdbc:postgresql://127.0.0.1:19003/binary_storage_test @@ -438,7 +469,7 @@ stack: ```sh ./mvnw install -pl :dotcms-core --am -DskipTests -Ddocker.skip ./mvnw verify -pl :dotcms-integration -Dmaven.build.cache.enabled=false -Dcoreit.test.skip=false \ - -Dit.test=BinaryAssetStorageIntegrationTest,ContentletBackupStorageTest,SharedAssetStorageIntegrationTest,BinaryAssetStarterRestoreTest,PublishingArchiveStorageTest + -Dit.test=BinaryAssetStorageIntegrationTest,ContentletBackupStorageTest,SharedAssetStorageIntegrationTest,BinaryAssetStarterRestoreTest,PublishingArchiveStorageTest,DotWebdavHelperTest ``` These default to flag-off mode, where the S3 cases are skipped. To run the S3 cases, create a diff --git a/dotCMS/src/main/java/com/dotcms/rest/api/v1/temp/TempFileAPI.java b/dotCMS/src/main/java/com/dotcms/rest/api/v1/temp/TempFileAPI.java index 15ff0f3cdad0..85f33761402b 100644 --- a/dotCMS/src/main/java/com/dotcms/rest/api/v1/temp/TempFileAPI.java +++ b/dotCMS/src/main/java/com/dotcms/rest/api/v1/temp/TempFileAPI.java @@ -3,6 +3,8 @@ import com.dotcms.http.CircuitBreakerUrl; import com.dotcms.http.CircuitBreakerUrl.Method; import com.dotcms.rest.exception.BadRequestException; +import com.dotcms.storage.AssetStorageFeature; +import com.dotcms.storage.TemporaryAssetStorage; import com.dotcms.util.CloseUtils; import com.dotcms.util.ConversionUtils; import com.dotcms.util.SecurityUtils; @@ -112,7 +114,14 @@ public DotTempFile createEmptyTempFile(final String incomingFileName,final HttpS final String tempFileId = TEMP_RESOURCE_PREFIX + UUIDGenerator.shorty(); final String tempFileUri = File.separator + tempFileId + File.separator + incomingFileName; - final File tempFile = new File(APILocator.getFileAssetAPI().getRealAssetPathTmpBinary() + tempFileUri); + final File tempFile; + try { + tempFile = AssetStorageFeature.isEnabled() + ? TemporaryAssetStorage.getInstance().file(tempFileId, incomingFileName) + : new File(APILocator.getFileAssetAPI().getRealAssetPathTmpBinary() + tempFileUri); + } catch (IOException e) { + throw new DotRuntimeException("Invalid temporary upload path", e); + } final File tempFolder = tempFile.getParentFile(); if (!tempFolder.mkdirs()) { @@ -126,6 +135,14 @@ public DotTempFile createEmptyTempFile(final String incomingFileName,final HttpS throw new DotRuntimeException("Invalid file upload"); } createTempPermissionFile(tempFolder, allowList); + if (AssetStorageFeature.isEnabled()) { + try { + Files.writeString(java.nio.file.Path.of(com.dotmarketing.util.ConfigUtils.getAssetTempPath(), + tempFileId, TemporaryAssetStorage.MANAGED_MARKER), ""); + } catch (IOException e) { + throw new DotRuntimeException("Unable to mark temporary upload", e); + } + } SecurityLogger.logInfo(this.getClass(),"Temp File Created with id: " + tempFileId + ", uploaded by userId: " + user.getUserId()); return new DotTempFile(tempFileId, tempFile); } @@ -174,7 +191,8 @@ public DotTempFile createTempFile(final String incomingFileName,final HttpServle final File tempFile = dotTempFile.file; final long maxLength = maxFileSize(request); - try (final OutputStream out = new BoundedOutputStream(maxLength,Files.newOutputStream(tempFile.toPath()))) { + try { + try (final OutputStream out = new BoundedOutputStream(maxLength,Files.newOutputStream(tempFile.toPath()))) { int read = 0; @@ -182,12 +200,12 @@ public DotTempFile createTempFile(final String incomingFileName,final HttpServle while ((read = inputStream.read(bytes)) != -1) { out.write(bytes, 0, read); } - - if (dotTempFile.metadata == null && dotTempFile.file.exists()) { - - return new DotTempFile(dotTempFile.id, dotTempFile.file); + if (!AssetStorageFeature.isEnabled()) { + return dotTempFile.metadata == null && dotTempFile.file.exists() + ? new DotTempFile(dotTempFile.id, dotTempFile.file) : dotTempFile; + } } - return dotTempFile; + return completeTempFile(dotTempFile); } catch (IOException e) { final String message = APILocator.getLanguageAPI().getStringKey(WebAPILocator.getLanguageWebAPI().getLanguage(request), "temp.file.max.file.size.error").replace("{0}", UtilMethods.prettyByteify(maxLength)); throw new DotStateException(message, e); @@ -198,6 +216,24 @@ public DotTempFile createTempFile(final String incomingFileName,final HttpServle } } + /** Finish a closed upload, including files written by the image editor. */ + public DotTempFile completeTempFile(final DotTempFile temporary) { + DotTempFile completed = temporary.metadata == null && temporary.file.exists() + ? new DotTempFile(temporary.id, temporary.file) : temporary; + if (AssetStorageFeature.isEnabled()) { + // DotTempFile can add a detected extension while building its metadata. + if (!completed.file.exists() && !completed.file.getName().equals(completed.fileName)) { + completed = new DotTempFile(completed.id, new File(completed.file.getParentFile(), completed.fileName)); + } + try { + TemporaryAssetStorage.getInstance().store(completed.id, completed.file); + } catch (com.dotmarketing.exception.DotDataException e) { + throw new DotRuntimeException("Unable to complete temporary upload", e); + } + } + return completed; + } + /** * Takes a URL, downloads it and the returns the resulting file as tempFile with a unique ID and * file handle that can be used to access the temp file. The request will be used to create a @@ -246,11 +282,7 @@ public DotTempFile createTempFileFromUrl(final String incomingFileName, urlGetter.doOut(out); } - if (dotTempFile.metadata == null && dotTempFile.file.exists()) { - - return new DotTempFile(dotTempFile.id, dotTempFile.file); - } - return dotTempFile; + return completeTempFile(dotTempFile); } /** @@ -360,6 +392,19 @@ private boolean canUseTempFile(final List incomingAccessingList, final D * @return */ public Optional getTempFile(final List accessingList, final String tempFileId) { + if (AssetStorageFeature.isEnabled()) { + if (!TemporaryAssetStorage.validId(tempFileId)) return Optional.empty(); + try { + final var store = TemporaryAssetStorage.getInstance(); + final var record = store.receipt(tempFileId); + if (record.isPresent()) { + return store.retrieve(record.get(), accessingList).map(file -> new DotTempFile(tempFileId, file)); + } + if (store.isManagedLocally(tempFileId)) return Optional.empty(); + } catch (com.dotmarketing.exception.DotDataException e) { + throw new DotRuntimeException("Unable to retrieve temporary upload", e); + } + } Optional tempFile = getTempFile(tempFileId); if (tempFile.isPresent() && canUseTempFile(accessingList, tempFile.get())) { return tempFile; @@ -405,6 +450,17 @@ public Optional getTempFile(final HttpServletRequest request, final * @return */ public boolean isTempResource(final String tempFileId) { + if (AssetStorageFeature.isEnabled()) { + if (!TemporaryAssetStorage.validId(tempFileId)) return false; + try { + final var store = TemporaryAssetStorage.getInstance(); + final var record = store.receipt(tempFileId); + if (record.isPresent()) return store.exists(record.get()); + if (store.isManagedLocally(tempFileId)) return false; + } catch (com.dotmarketing.exception.DotDataException e) { + throw new DotRuntimeException("Unable to check temporary upload", e); + } + } return getTempFile(tempFileId).isPresent(); } @@ -456,6 +512,13 @@ public String getRequestFingerprint(final HttpServletRequest request) { */ public Optional getTempResourceId(final File file){ try { + if (AssetStorageFeature.isEnabled()) { + final var root = new File(com.dotmarketing.util.ConfigUtils.getAssetTempPath()).getCanonicalFile().toPath(); + final var path = file.getCanonicalFile().toPath(); + if (!path.startsWith(root) || root.relativize(path).getNameCount() < 2) return Optional.empty(); + final String id = root.relativize(path).getName(0).toString(); + return isTempResource(id) ? Optional.of(id) : Optional.empty(); + } final String tempResourceId = file.toPath().getParent().getFileName().toString(); if (isTempResource(tempResourceId)) { return Optional.of(tempResourceId); diff --git a/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java b/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java index 5b824420a986..d69a9e74f83d 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java @@ -1065,6 +1065,22 @@ private String tempResourcePath(final String tempResourceId){ return ConfigUtils.getAssetTempPath() + File.separator + tempResourceId + File.separator + tempResourceId + META_TMP; } + private StorageKey temporaryMetadataKey(final String id, final boolean legacy) { + if (!TemporaryAssetStorage.validId(id)) { + throw new IllegalArgumentException("Invalid temporary resource id"); + } + return new StorageKey.Builder().group(Config.getStringProperty(METADATA_GROUP_NAME, DOT_METADATA)) + .path(legacy ? tempResourcePath(id) : TemporaryAssetStorage.metadataPath(id)) + .storage(legacy ? StorageType.FILE_SYSTEM : StorageType.S3).build(); + } + + private Map temporaryMetadata(final String id) throws DotDataException { + // Temporary metadata is mutable: read S3 directly so another node's edits are visible. + // Only a genuine absence falls back to pre-feature local metadata, never a storage failure. + final Map shared = fileStorageAPI.retrieveRawMetaData(temporaryMetadataKey(id, false)); + return shared != null ? shared : fileStorageAPI.retrieveRawMetaData(temporaryMetadataKey(id, true)); + } + /** * {@inheritDoc} * @param tempResourceId @@ -1074,10 +1090,34 @@ private String tempResourcePath(final String tempResourceId){ public void putCustomMetadataAttributes(final String tempResourceId, final Map> customAttributesByField) throws DotDataException { + if (AssetStorageFeature.isEnabled()) { + if (customAttributesByField.isEmpty()) { + return; + } + final Map previous = temporaryMetadata(tempResourceId); + final Map updated = new HashMap<>(previous == null ? Map.of() : previous); + for (final Map attributes : customAttributesByField.values()) { + if (attributes.isEmpty()) { + updated.keySet().removeIf(key -> key.startsWith(Metadata.CUSTOM_PROP_PREFIX)); + } else { + attributes.forEach((key, value) -> updated.put(Metadata.CUSTOM_PROP_PREFIX + key, value)); + } + } + // Retain a nonempty record after clearing custom attributes, so a legacy local copy + // cannot resurrect an earlier focal point on the next read. + updated.put("tempResourceId", tempResourceId); + if (!fileStorageAPI.setMetadata(new FetchMetadataParams.Builder().cache(false) + .storageKey(temporaryMetadataKey(tempResourceId, false)).build(), updated)) { + throw new DotDataException("Unable to save temporary binary metadata for " + tempResourceId); + } + return; + } + final String metadataBucketName = Config .getStringProperty(METADATA_GROUP_NAME, DOT_METADATA); - customAttributesByField.forEach((fieldName, customAttributes) -> { + for (final var entry : customAttributesByField.entrySet()) { + final Map customAttributes = entry.getValue(); try { final String tempResourcePath = tempResourcePath(tempResourceId); @@ -1092,9 +1132,12 @@ public void putCustomMetadataAttributes(final String tempResourceId, .build()), customAttributes); }catch (Exception e){ + if (AssetStorageFeature.isEnabled()) { + throw new DotDataException("Unable to save temporary binary metadata for " + tempResourceId, e); + } Logger.error(FileMetadataAPIImpl.class, "Error saving custom attributes", e); } - }); + } } /** @@ -1106,6 +1149,11 @@ public void putCustomMetadataAttributes(final String tempResourceId, public Optional getMetadata(final String tempResourceId) throws DotDataException { + if (AssetStorageFeature.isEnabled()) { + return Optional.ofNullable(temporaryMetadata(tempResourceId)) + .map(values -> new Metadata(tempResourceId, values)); + } + final StorageType storageType = StoragePersistenceProvider.getStorageType(); final String metadataBucketName = Config .getStringProperty(METADATA_GROUP_NAME, DOT_METADATA); diff --git a/dotCMS/src/main/java/com/dotcms/storage/FileSystemStoragePersistenceAPIImpl.java b/dotCMS/src/main/java/com/dotcms/storage/FileSystemStoragePersistenceAPIImpl.java index 8d3ee9241cf6..5e31fbe38c17 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/FileSystemStoragePersistenceAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/storage/FileSystemStoragePersistenceAPIImpl.java @@ -100,7 +100,8 @@ private String normalizePath(final String groupName, final String path) { return AssetStorageFeature.isEnabled() && (com.dotcms.storage.binary.BinaryAssetStorageAPI.BINARY_ASSETS_GROUP.equalsIgnoreCase(groupName) || com.dotcms.storage.binary.BinaryAssetStorageAPI.GENERATED_ASSETS_GROUP.equalsIgnoreCase(groupName) - || com.dotcms.publishing.output.BundleArchiveStorage.GROUP.equalsIgnoreCase(groupName)) + || com.dotcms.publishing.output.BundleArchiveStorage.GROUP.equalsIgnoreCase(groupName) + || TemporaryAssetStorage.GROUP.equalsIgnoreCase(groupName)) ? path : path.toLowerCase(); } diff --git a/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java b/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java new file mode 100644 index 000000000000..57271addab6b --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java @@ -0,0 +1,254 @@ +package com.dotcms.storage; + +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.ConfigUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** Immutable temporary uploads, with access and original expiry checked before downloading bytes. */ +public final class TemporaryAssetStorage { + public static final String GROUP = "temporary-assets"; + public static final String MANAGED_MARKER = ".s3-upload"; + public static final String PERMISSIONS_FILE = "whoCanUse.tmp"; + private static final String RECEIPTS = ".receipts/"; + private static final ObjectMapper JSON = new ObjectMapper(); + private static final JsonReaderDelegate READER = new JsonReaderDelegate<>(Map.class); + private StoragePersistenceAPI remote; + private FileSystemStoragePersistenceAPIImpl local; + private Path root; + + private static class Holder { + private static final TemporaryAssetStorage INSTANCE = new TemporaryAssetStorage(null, null); + } + + public static TemporaryAssetStorage getInstance() { + return Holder.INSTANCE; + } + + public TemporaryAssetStorage(final StoragePersistenceAPI remote, final Path root) { + this.remote = remote; + this.root = root; + } + + public static boolean validId(final String id) { + return id != null && id.matches("temp_[A-Za-z0-9_-]{1,128}"); + } + + public static String metadataPath(final String id) { + if (!validId(id)) throw new IllegalArgumentException("Invalid temporary resource id"); + return "/tmp_upload/" + id + "/" + id + FileMetadataAPI.META_TMP; + } + + private Path root() { + return root == null ? Path.of(ConfigUtils.getAssetTempPath()) : root; + } + + /** Validates the upload's own directory before any producer creates directories or writes bytes. */ + public File file(final String id, final String name) throws IOException { + if (!validId(id) || name == null || name.isBlank() || name.contains("\\")) { + throw new IllegalArgumentException("Invalid temporary upload path"); + } + final Path owner = root().toFile().getCanonicalFile().toPath().resolve(id); + final Path requested = owner.resolve(name); + if (!requested.startsWith(owner) || Files.isSymbolicLink(owner)) { + throw new IllegalArgumentException("Invalid temporary upload directory"); + } + Path component = owner; + for (Path part : owner.relativize(requested)) { + component = component.resolve(part); + if (Files.isSymbolicLink(component)) { + throw new IllegalArgumentException("Invalid temporary upload link"); + } + } + final Path target = requested.toFile().getCanonicalFile().toPath(); + if (!target.startsWith(owner) || target.equals(owner)) { + throw new IllegalArgumentException("Temporary upload is outside its own directory"); + } + for (Path part : owner.relativize(target)) { + if (part.toString().startsWith(".") || part.toString().equalsIgnoreCase(PERMISSIONS_FILE) + || part.toString().endsWith(FileMetadataAPI.META_TMP)) { + throw new IllegalArgumentException("Reserved temporary upload path"); + } + } + return target.toFile(); + } + + public boolean isManagedLocally(final String id) { + return validId(id) && Files.exists(root().resolve(id).resolve(MANAGED_MARKER)); + } + + private synchronized StoragePersistenceAPI remote() throws DotDataException { + if (remote == null) { + remote = AmazonS3StoragePersistenceAPIImpl.withPlainPaths(); + } + return remote; + } + + private synchronized FileSystemStoragePersistenceAPIImpl local() throws IOException { + if (local == null) { + Files.createDirectories(root()); + local = new FileSystemStoragePersistenceAPIImpl(); + local.addGroupMapping(GROUP, root().toFile()); + } + return local; + } + + public record Receipt(String id, String name, long modifiedAt, List allowed) { + public boolean expired() { + final long ageMillis = Config.getIntProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", 1800) * 1000L; + return modifiedAt <= System.currentTimeMillis() - ageMillis; + } + + String prefix() { + return "data/" + modifiedAt + "/" + id + "/"; + } + + String key() { + return prefix() + name; + } + } + + public Optional receipt(final String id) throws DotDataException { + if (!AssetStorageFeature.isEnabled() || !validId(id)) return Optional.empty(); + final Map record = (Map) remote().pullObject(GROUP, RECEIPTS + id + ".json", READER); + if (record == null) return Optional.empty(); + try { + final String name = (String) record.get("name"); + file(id, name); + final long modified = ((Number) record.get("modifiedAt")).longValue(); + if (modified <= 0) throw new IllegalArgumentException("Invalid upload timestamp"); + final List allowed = ((List) record.get("allowed")).stream() + .map(value -> (String) value).toList(); + return Optional.of(new Receipt(id, name, modified, List.copyOf(allowed))); + } catch (Exception invalid) { + throw new DotDataException("Invalid temporary upload record for " + id, invalid); + } + } + + /** Called only after the upload stream has closed and its final filename has been resolved. */ + public void store(final String id, final File source) throws DotDataException { + if (!AssetStorageFeature.isEnabled()) return; + try { + final Path owner = root().toFile().getCanonicalFile().toPath().resolve(id); + final String name = owner.relativize(source.getCanonicalFile().toPath()).toString().replace(File.separatorChar, '/'); + final File validated = file(id, name); + if (!validated.isFile()) throw new IOException("Temporary upload is incomplete"); + Files.writeString(owner.resolve(MANAGED_MARKER), ""); + final File permissions = new File(validated.getParentFile(), PERMISSIONS_FILE); + if (Files.isSymbolicLink(permissions.toPath())) throw new IOException("Invalid upload permissions path"); + final List allowed = List.copyOf(JSON.readValue(permissions, List.class)); + final Receipt record = receipt(id).orElseGet(() -> new Receipt(id, name, validated.lastModified(), allowed)); + if (!record.name().equals(name) || !record.allowed().equals(allowed) || record.expired()) { + throw new DotDataException("Temporary upload changed or expired: " + id); + } + remote().createGroup(GROUP); + // Record ownership/expiry first, so a failed upload is still discoverable for cleanup. + final var data = new java.util.HashMap(); + data.put("name", record.name()); + data.put("modifiedAt", record.modifiedAt()); + data.put("allowed", new java.util.ArrayList<>(record.allowed())); + if (!remote().backfillObject(GROUP, RECEIPTS + id + ".json", new JsonWriterDelegate(), READER, data) + || !remote().backfillFile(GROUP, record.key(), validated)) { + throw new DotDataException("Temporary upload has no verified S3 copy: " + id); + } + } catch (IOException | RuntimeException failure) { + throw new DotDataException("Unable to store temporary upload " + id, failure); + } + } + + public boolean exists(final Receipt record) throws DotDataException { + return AssetStorageFeature.isEnabled() && !record.expired() && remote().existsObject(GROUP, record.key()); + } + + public Optional retrieve(final Receipt record, final List accessing) throws DotDataException { + if (!AssetStorageFeature.isEnabled() || record.expired() || accessing == null + || Collections.disjoint(record.allowed(), accessing)) return Optional.empty(); + try { + final File target = file(record.id(), record.name()); + if (!target.isFile()) { + final File downloaded = remote().pullFile(GROUP, record.key()); + if (downloaded == null) return Optional.empty(); + try { + // The marker prevents legacy fallback if an expired/deleted receipt disappears. + Files.createDirectories(root().resolve(record.id())); + Files.writeString(root().resolve(record.id()).resolve(MANAGED_MARKER), ""); + local().pushFile(GROUP, record.id() + "/" + record.name(), downloaded, Map.of()); + Files.setLastModifiedTime(target.toPath(), java.nio.file.attribute.FileTime.fromMillis(record.modifiedAt())); + } finally { + remote().releaseRetrievedFile(downloaded); + } + } + final Path permissions = target.toPath().getParent().resolve(PERMISSIONS_FILE); + if (Files.isSymbolicLink(permissions)) throw new IOException("Invalid upload permissions path"); + if (!Files.exists(permissions)) JSON.writeValue(permissions.toFile(), record.allowed()); + // Restoration must not renew the logical TTL; it lives in the immutable receipt. + return record.expired() ? Optional.empty() : Optional.of(target); + } catch (IOException | RuntimeException failure) { + throw new DotDataException("Unable to restore temporary upload " + record.id(), failure); + } + } + + private void removeMetadataAndRenditions(final String id) throws DotDataException { + APILocator.getFileStorageAPI().removeMetaData(new FetchMetadataParams.Builder().cache(false) + .storageKey(new StorageKey.Builder().storage(StorageType.S3) + .group(Config.getStringProperty(StoragePersistenceProvider.METADATA_GROUP_NAME, FileMetadataAPI.DOT_METADATA)) + .path(metadataPath(id)).build()).build()); + APILocator.getBinaryAssetStorageAPI().deleteGeneratedFiles(id); + } + + /** Timestamped payload prefixes also expose interrupted uploads after their receipt is gone. */ + public void cleanupExpired() throws DotDataException { + if (!AssetStorageFeature.isEnabled()) return; + for (String path : remote().listObjectPaths(GROUP, RECEIPTS)) { + if (!path.startsWith(RECEIPTS) || !path.endsWith(".json")) continue; + final String id = path.substring(RECEIPTS.length(), path.length() - ".json".length()); + final var record = receipt(id); + if (record.isEmpty() || !record.get().expired()) continue; + removeMetadataAndRenditions(id); + delete(record.get().prefix()); + delete(path); + try { + final Path directory = file(id, "placeholder").toPath().getParent(); + com.liferay.util.FileUtil.deltree(directory.toFile()); + if (Files.exists(directory)) throw new IOException("Temporary cache directory remains"); + } catch (IOException failure) { + throw new DotDataException("Unable to remove expired temporary cache " + id, failure); + } + } + final var expired = new HashSet(); + for (String path : remote().listObjectPaths(GROUP, "data/")) { + final String[] parts = path.split("/", 4); + if (parts.length != 4 || !parts[0].equals("data") || !validId(parts[2])) continue; + try { + final var record = new Receipt(parts[2], parts[3], Long.parseLong(parts[1]), List.of()); + if (record.expired()) expired.add(record.prefix()); + } catch (NumberFormatException ignored) { + // Only this uploader's timestamped namespace belongs to this cleanup. + } + } + for (String prefix : expired) delete(prefix); + } + + private void delete(final String path) throws DotDataException { + if (path.endsWith("/")) { + for (String object : remote().listObjectPaths(GROUP, path)) { + delete(object); + } + return; + } + if (!remote().deleteObjectAndReferences(GROUP, path)) { + throw new DotDataException("Unable to remove expired temporary object " + path); + } + } +} diff --git a/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java b/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java new file mode 100644 index 000000000000..e1075b9680ac --- /dev/null +++ b/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java @@ -0,0 +1,341 @@ +package com.dotcms.storage; + +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.ConfigUtils; +import java.io.File; +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** WebDAV names point to completed immutable payloads; local files are disposable copies. */ +public final class WebdavTemporaryStorage { + public static final String GROUP = "webdav-temporary"; + private static final JsonReaderDelegate READER = new JsonReaderDelegate<>(Map.class); + private final Path configuredRoot; + private AmazonS3StoragePersistenceAPIImpl remote; + private boolean groupReady; + + private static class Holder { + private static final WebdavTemporaryStorage INSTANCE = new WebdavTemporaryStorage(null, null); + } + + public static WebdavTemporaryStorage getInstance() { return Holder.INSTANCE; } + + public WebdavTemporaryStorage(AmazonS3StoragePersistenceAPIImpl remote, Path root) { + this.remote = remote; + this.configuredRoot = root; + } + + private Path root() throws IOException { + return (configuredRoot == null ? Path.of(ConfigUtils.getAssetTempPath()) : configuredRoot) + .toFile().getCanonicalFile().toPath(); + } + + public String key(File file) throws IOException { + final Path root = root(); + final Path supplied = file.toPath().toAbsolutePath().normalize(); + final Path configured = (configuredRoot == null ? Path.of(ConfigUtils.getAssetTempPath()) : configuredRoot) + .toAbsolutePath().normalize(); + final Path boundary = supplied.startsWith(configured) ? configured : root; + if (!supplied.startsWith(boundary)) throw new IOException("Invalid WebDAV staging path"); + for (Path part = supplied; !part.equals(boundary); part = part.getParent()) { + if (Files.isSymbolicLink(part)) throw new IOException("WebDAV staging path contains a symbolic link"); + } + final Path canonical = file.getCanonicalFile().toPath(); + if (!canonical.startsWith(root) || canonical.equals(root)) throw new IOException("Invalid WebDAV staging path"); + final String key = root.relativize(canonical).toString().replace(File.separatorChar, '/'); + if (key.startsWith(".webdav-cache/") || key.equals(".webdav-cache")) throw new IOException("Reserved staging path"); + return key; + } + + public File file(String key) throws IOException { + final Path path = Path.of(key); + if (path.isAbsolute() || key.contains("\\") || !path.normalize().equals(path)) throw new IOException("Invalid staging key"); + final File file = root().resolve(path).toFile(); + if (!key(file).equals(key)) throw new IOException("Invalid staging key"); + return file; + } + + private synchronized AmazonS3StoragePersistenceAPIImpl remote() throws DotDataException { + if (!AssetStorageFeature.isEnabled()) throw new DotDataException("S3 asset storage is disabled"); + if (remote == null) remote = AmazonS3StoragePersistenceAPIImpl.withPlainPaths(); + if (!groupReady) { + remote.createGroup(GROUP); + groupReady = true; + } + return remote; + } + + public record Entry(String path, boolean directory, long size, long modified, String dataKey) { + public boolean expired() { + return modified <= System.currentTimeMillis() + - Config.getIntProperty("CLEANUP_TMP_FILES_OLDER_THAN_HOURS", 3) * 3_600_000L; + } + } + + // A record is a leaf beside its descendants, including on S3-compatible filesystem backends. + private String recordPrefix(String path) { + return "entries/" + path.replace("%", "%25").replace(".", "%2E") + "/"; + } + + private String recordPath(String object) throws IOException { + if (!object.startsWith("entries/") || !object.endsWith("/.entry")) { + throw new IOException("Invalid WebDAV entry key"); + } + final String path = object.substring("entries/".length(), object.length() - "/.entry".length()) + .replace("%2E", ".").replace("%25", "%"); + file(path); + if (!(recordPrefix(path) + ".entry").equals(object)) throw new IOException("Invalid WebDAV entry key"); + return path; + } + + private record State(AmazonS3StoragePersistenceAPIImpl.ObjectSnapshot snapshot, Entry entry, String pending) { } + + private State state(String key) throws DotDataException, IOException { + file(key); + final var object = remote().readObjectSnapshot(GROUP, recordPrefix(key) + ".entry", READER); + if (object == null) return null; + try { + final Map data = (Map) object.value(); + final String pending = (String) data.get("pending"); + if (pending != null) validateDataKey(key, pending); + Entry entry = null; + if (data.containsKey("directory")) { + entry = new Entry(key, (Boolean) data.get("directory"), ((Number) data.get("size")).longValue(), + ((Number) data.get("modified")).longValue(), (String) data.get("dataKey")); + if (entry.size() < 0 || entry.modified() <= 0 || entry.dataKey() == null) { + throw new IOException("Invalid WebDAV entry: " + key); + } + if (!entry.directory()) validateDataKey(key, entry.dataKey()); + } + return new State(object, entry, pending); + } catch (RuntimeException failure) { + throw new IOException("Invalid WebDAV entry: " + key, failure); + } + } + + private void validateDataKey(String path, String dataKey) throws IOException { + if (!dataKey.matches("data/[0-9]+/[0-9a-f-]{36}/[^/]+") + || !Path.of(dataKey).getFileName().equals(Path.of(path).getFileName()) + || !Path.of(dataKey).normalize().toString().equals(dataKey)) { + throw new IOException("Invalid WebDAV payload reference: " + path); + } + } + + private Entry read(String key) throws DotDataException, IOException { + final State state = state(key); + return state == null ? null : state.entry(); + } + + public Entry stat(File file) { + try { + final String key = key(file); + final Entry entry = read(key); + if (entry != null && !entry.expired()) return entry; + // Parents can be implicit, including after a concurrent child write and folder deletion. + final var children = children(file); + return children.isEmpty() ? null : new Entry(key, true, 0, + children.stream().mapToLong(Entry::modified).max().orElseThrow(), ""); + } catch (IOException | DotDataException failure) { + throw new DotRuntimeException("Unable to inspect WebDAV staging file", failure); + } + } + + public List children(File directory) throws IOException, DotDataException { + final String prefix = key(directory) + "/"; + final Map children = new java.util.TreeMap<>(); + for (String object : remote().listObjectPaths(GROUP, recordPrefix(key(directory)))) { + final String path = recordPath(object); + if (path.equals(key(directory))) continue; + if (!path.startsWith(prefix)) throw new IOException("Invalid staging listing"); + final Entry entry = read(path); + if (entry == null || entry.expired()) continue; + final String suffix = path.substring(prefix.length()); + final int slash = suffix.indexOf('/'); + if (slash < 0) children.put(path, entry); + else { + final String parent = prefix + suffix.substring(0, slash); + children.putIfAbsent(parent, new Entry(parent, true, 0, entry.modified(), "")); + } + } + return new ArrayList<>(children.values()); + } + + private String publish(String path, Entry entry, String pending, String expected) throws DotDataException { + final var data = new HashMap(); + // Every mutation changes the ETag, including rollback to the same completed entry. + data.put("mutation", UUID.randomUUID().toString()); + if (entry != null) { + data.put("directory", entry.directory()); + data.put("size", entry.size()); + data.put("modified", entry.modified()); + data.put("dataKey", entry.dataKey()); + } + if (pending != null) data.put("pending", pending); + final String version = remote().writeObjectIfMatch(GROUP, recordPrefix(path) + ".entry", data, expected); + if (version == null) throw new DotDataException("WebDAV path changed concurrently: " + path); + return version; + } + + public void mkdir(File directory) throws IOException, DotDataException { + final String path = key(directory); + final State current = state(path); + final Entry existing = current == null ? null : current.entry(); + if (existing != null && !existing.directory() && !existing.expired()) throw new IOException("A file exists at " + path); + publish(path, new Entry(path, true, 0, System.currentTimeMillis(), ""), null, + current == null ? null : current.snapshot().version()); + } + + /** Reserve a reference before uploading; cleanup or a competing writer fences late publication. */ + public void store(File destination, File completed) throws IOException, DotDataException { + final String path = key(destination); + final State before = state(path); + final Entry existing = before == null ? null : before.entry(); + if ((existing != null && existing.directory() && !existing.expired()) || !children(destination).isEmpty()) { + throw new IOException("A directory exists at " + path); + } + final String dataKey = "data/" + System.currentTimeMillis() + "/" + UUID.randomUUID() + "/" + destination.getName(); + final String reservation = publish(path, existing, dataKey, before == null ? null : before.snapshot().version()); + try { + if (!remote().backfillFile(GROUP, dataKey, completed)) throw new IOException("WebDAV payload was not verified in S3"); + publish(path, new Entry(path, false, Files.size(completed.toPath()), System.currentTimeMillis(), dataKey), null, reservation); + } catch (IOException | DotDataException | RuntimeException failure) { + try { + // Restore only our own reservation; never replace a newer upload or resurrect a deletion. + publish(path, existing, null, reservation); + } catch (Exception rollbackFailure) { + failure.addSuppressed(rollbackFailure); + } + throw failure; + } + } + + /** Immutable cache names prevent another upload from changing an in-flight response. */ + public File materialize(Entry entry) throws IOException, DotDataException { + if (entry == null || entry.directory() || entry.expired()) throw new IOException("Missing WebDAV staging file"); + file(entry.path()); + final Path data = Path.of(entry.dataKey()); + if (data.isAbsolute() || !data.normalize().equals(data) || !entry.dataKey().startsWith("data/")) { + throw new IOException("Invalid WebDAV payload key"); + } + final Path cache = root().resolve(".webdav-cache").resolve(data); + if (!cache.toFile().getCanonicalFile().toPath().equals(cache)) throw new IOException("Invalid WebDAV cache path"); + if (!Files.isRegularFile(cache)) { + final File download = remote().pullFile(GROUP, entry.dataKey()); + if (download == null) throw new IOException("Missing WebDAV payload: " + entry.path()); + try { + Files.createDirectories(cache.getParent()); + final Path stage = Files.createTempFile(cache.getParent(), ".download-", ".tmp"); + try { + Files.copy(download.toPath(), stage, StandardCopyOption.REPLACE_EXISTING); + if (Files.size(stage) != entry.size()) throw new IOException("Incomplete WebDAV payload"); + Files.move(stage, cache, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } finally { + Files.deleteIfExists(stage); + } + } finally { + remote().releaseRetrievedFile(download); + } + } + return cache.toFile(); + } + + public void delete(File target) throws IOException, DotDataException { + final String key = key(target); + final var objects = new ArrayList<>(remote().listObjectPaths(GROUP, recordPrefix(key))); + objects.sort(Comparator.reverseOrder()); + + for (String object : objects) { + final String path = recordPath(object); + if (!path.equals(key) && !path.startsWith(key + "/")) throw new IOException("Invalid staging deletion path"); + final State current = state(path); + if (current == null) continue; + publish(path, null, null, current.snapshot().version()); + final Entry entry = current.entry(); + if (entry != null && !entry.directory()) remove(entry.dataKey()); + // A pending upload may still be running. Its reservation is gone, so it cannot publish; + // its completed payload, if any, is reclaimed by the reference-aware expiry pass. + } + com.liferay.util.FileUtil.deltree(target, true); + } + + /** Uses a server timestamp so node clock skew cannot age a newly created reservation. */ + private long cleanupCutoff() throws DotDataException { + final int hours = Config.getIntProperty("CLEANUP_TMP_FILES_OLDER_THAN_HOURS", 3); + if (hours < 0) throw new DotDataException("WebDAV cleanup age must not be negative"); + final String path = "maintenance/" + UUID.randomUUID(); + final String version = remote().writeObjectIfMatch(GROUP, path, new HashMap(), null); + if (version == null) throw new DotDataException("Unable to establish S3 cleanup time"); + try { + final var clock = remote().readObjectSnapshot(GROUP, path, READER); + if (clock == null || !version.equals(clock.version())) throw new DotDataException("S3 cleanup clock changed"); + return clock.modified() - hours * 3_600_000L; + } finally { + remote().deleteObjectAndReferences(GROUP, path); + } + } + + /** Fence expired reservations with conditional tombstones, then collect aged, unreferenced payloads. */ + public void cleanupExpired() throws DotDataException { + if (!AssetStorageFeature.isEnabled()) return; + final long cutoff = cleanupCutoff(); + final var references = new java.util.HashSet(); + try { + for (String object : remote().listObjectPaths(GROUP, "entries/")) { + final State current = state(recordPath(object)); + if (current == null) continue; + if (current.entry() == null && current.pending() == null) continue; + if (current.snapshot().modified() < cutoff) { + // Never DELETE a mutable record: some S3-compatible stores ignore If-Match on DELETE. + // A unique tombstone fences late writers. A conflicting publication aborts this pass. + publish(recordPath(object), null, null, current.snapshot().version()); + } else { + if (current.entry() != null && !current.entry().directory()) references.add(current.entry().dataKey()); + if (current.pending() != null) references.add(current.pending()); + } + } + // New reservations are written before payload creation. Payloads published after this + // scan are either in references, younger than cutoff, or fenced by a tombstoned reservation. + for (var object : remote().listObjectSnapshots(GROUP, "data/")) { + if (!object.path().matches("data/[0-9]+/[0-9a-f-]{36}/[^/]+")) { + throw new IOException("Invalid WebDAV payload inventory"); + } + if (object.modified() < cutoff && !references.contains(object.path())) { + // Payload keys are unique and immutable. Every publisher first reserves its reference. + remove(object.path()); + } + } + for (var object : remote().listObjectSnapshots(GROUP, "maintenance/")) { + if (object.modified() < cutoff) remove(object.path()); + } + } catch (IOException failure) { + throw new DotDataException("Unable to clean WebDAV staging objects", failure); + } + } + + private void remove(String key) throws DotDataException { + if (!remote().deleteObjectAndReferences(GROUP, key)) throw new DotDataException("Unable to delete WebDAV object " + key); + } + + public void copy(File source, File destination) throws IOException, DotDataException { + final String from = key(source), to = key(destination); + if (from.equals(to)) return; + if (to.startsWith(from + "/")) throw new IOException("Cannot copy a staging directory into itself"); + final Entry entry = stat(source); + if (entry == null) throw new IOException("Missing WebDAV copy source"); + if (entry.directory()) { + mkdir(destination); + for (Entry child : children(source)) copy(file(child.path()), new File(destination, Path.of(child.path()).getFileName().toString())); + } else store(destination, materialize(entry)); + } +} diff --git a/dotCMS/src/main/java/com/dotmarketing/quartz/job/BinaryCleanupJob.java b/dotCMS/src/main/java/com/dotmarketing/quartz/job/BinaryCleanupJob.java index 5fa5b6f16433..43fd28ccb6ec 100644 --- a/dotCMS/src/main/java/com/dotmarketing/quartz/job/BinaryCleanupJob.java +++ b/dotCMS/src/main/java/com/dotmarketing/quartz/job/BinaryCleanupJob.java @@ -69,6 +69,18 @@ public void execute(final JobExecutionContext ctx) throws JobExecutionException Logger.info(this.getClass(), "STARTING TMP/TRASH FILE CLEANUP"); cleanUpTmpUploadedFiles(); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + com.dotcms.storage.TemporaryAssetStorage.getInstance().cleanupExpired(); + } catch (com.dotmarketing.exception.DotDataException e) { + Logger.error(this.getClass(), "Unable to clean expired S3 temporary uploads; next cleanup will retry", e); + } + try { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().cleanupExpired(); + } catch (com.dotmarketing.exception.DotDataException e) { + Logger.error(this.getClass(), "Unable to clean expired S3 WebDAV uploads; next cleanup will retry", e); + } + } cleanUpOldBundles(); cleanUpTrashFolder(); cleanUpLocalBackupDirectory(); diff --git a/dotCMS/src/main/java/com/dotmarketing/servlets/BinaryExporterServlet.java b/dotCMS/src/main/java/com/dotmarketing/servlets/BinaryExporterServlet.java index 9bac5fd656b1..b1944ee48462 100644 --- a/dotCMS/src/main/java/com/dotmarketing/servlets/BinaryExporterServlet.java +++ b/dotCMS/src/main/java/com/dotmarketing/servlets/BinaryExporterServlet.java @@ -459,6 +459,9 @@ private void serveBinary(HttpServletRequest req, HttpServletResponse resp, final // TempFileAPI.createTempFile) so consumers get a usable image preview. if (temp.metadata == null && temp.file.exists()) { temp = new DotTempFile(temp.id, temp.file); + } + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + temp = tempFileAPI.completeTempFile(temp); } copyMetadata(uuid, fieldVarName, temp); // The focal point is authoritative in the request URL. Write it straight onto the saved diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/BasicFolderResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/BasicFolderResourceImpl.java index fc0dc4ebd263..9f0eaa15fa02 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/BasicFolderResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/BasicFolderResourceImpl.java @@ -92,8 +92,15 @@ private Resource createNewTemporalResource (final String newName, final InputStr try { this.originalPath = (!this.originalPath.endsWith("/"))? this.originalPath + "/":this.originalPath; - final File tempFile = this.dotDavHelper.createTempFile("/" + this.host.getHostname() + this.originalPath + newName); - FileUtils.copyStreamToFile(tempFile, in, null); + final String temporaryPath = com.dotcms.storage.AssetStorageFeature.isEnabled() + ? this.dotDavHelper.stripMapping(this.originalPath) + newName + : "/" + this.host.getHostname() + this.originalPath + newName; + final File tempFile = this.dotDavHelper.createTempFile(temporaryPath); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + DotWebdavHelper.writeCompletedTempFile(tempFile, in); + } else { + FileUtils.copyStreamToFile(tempFile, in, null); + } final Resource tempFileResource = new TempFileResourceImpl(tempFile, this.originalPath + newName, this.isAutoPub); return tempFileResource; } catch (Exception e){ diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java b/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java index c08834e0cb35..6f1ee1be9ecf 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java @@ -398,7 +398,36 @@ public File loadTempFile(String url){ Logger.error( this, "Error happened with uri: [" + url + "]", e); } Logger.debug(this, "Getting temp file from path " + url); - return new File(getTempDir().getPath() + url); + return new File(getTempDir().getPath() + temporaryPath(url)); + } + + /** CMS paths are case insensitive; the first temporary component and its descendants are not. */ + private String temporaryPath(String path) { + if (!com.dotcms.storage.AssetStorageFeature.isEnabled()) return path; + final String[] parts = path.split("/", -1); + for (int i = 0; i < parts.length; i++) { + if (isTempResource(parts[i])) break; + parts[i] = parts[i].toLowerCase(java.util.Locale.ROOT); + } + return String.join("/", parts); + } + + List temporaryChildren(String url, boolean autoPublish) throws IOException { + final var staging = com.dotcms.storage.WebdavTemporaryStorage.getInstance(); + final List result = new ArrayList<>(); + final String prefix = url.endsWith("/") ? url : url + "/"; + try { + for (var entry : staging.children(loadTempFile(url))) { + final File file = staging.file(entry.path()); + final String child = prefix + file.getName(); + if (!isTempResource(child)) continue; + result.add(entry.directory() ? new TempFolderResourceImpl(child, file, autoPublish) + : new TempFileResourceImpl(file, child, autoPublish)); + } + } catch (DotDataException failure) { + throw new IOException("Unable to list WebDAV staging resources", failure); + } + return result; } /** @@ -473,13 +502,15 @@ public List getChildrenOfFolder ( Folder parentFolder, User user, bool String p = APILocator.getIdentifierAPI().find(parentFolder.getIdentifier()).getPath(); if ( p.contains( "/" ) ) p.replace( "/", File.separator ); - File tempDir = new File( getTempDir().getPath() + File.separator + folderHost.getHostname() + p ); + File tempDir = new File( getTempDir().getPath() + temporaryPath(File.separator + folderHost.getHostname() + p) ); p = identifierAPI.find(parentFolder.getIdentifier()).getPath(); if ( !p.endsWith( "/" ) ) p = p + "/"; if ( !p.startsWith( "/" ) ) p = "/" + p; - if ( tempDir.exists() && tempDir.isDirectory() ) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + result.addAll(temporaryChildren(prePath + folderHost.getHostname() + p, isAutoPub)); + } else if ( tempDir.exists() && tempDir.isDirectory() ) { File[] files = tempDir.listFiles(); for ( File file : files ) { String tp = prePath + folderHost.getHostname() + p + file.getName(); @@ -534,8 +565,16 @@ public File createTempFolder(String path){ if(path.startsWith("/") || path.startsWith("\\")){ path = path.substring(1, path.length()); } + path = temporaryPath(path); path = path.replace("/", File.separator); File f = new File(getTempDir().getPath() + File.separator + path); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().mkdir(f); + } catch (IOException | DotDataException failure) { + throw new DotRuntimeException("Unable to create WebDAV staging directory", failure); + } + } f.mkdirs(); return f; } @@ -555,6 +594,18 @@ public void copyFolderToTemp(Folder folder, File tempFolder, User user, String n File tf = createTempFolder(path); List children = getChildrenOfFolder(folder, user, isAutoPub, lang); for (Resource resource : children) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled() + && (resource instanceof TempFolderResourceImpl || resource instanceof TempFileResourceImpl)) { + try { + final File source = resource instanceof TempFolderResourceImpl + ? ((TempFolderResourceImpl) resource).getFolder() + : ((TempFileResourceImpl) resource).getLogicalFile(); + com.dotcms.storage.WebdavTemporaryStorage.getInstance().copy(source, new File(tf, resource.getName())); + } catch (DotDataException failure) { + throw new IOException("Unable to copy WebDAV staging resource", failure); + } + continue; + } if(resource instanceof CollectionResource){ FolderResourceImpl fr = (FolderResourceImpl)resource; copyFolderToTemp(fr.getFolder(), tf, user, fr.getFolder().getName(),isAutoPub, lang); @@ -566,6 +617,16 @@ public void copyFolderToTemp(Folder folder, File tempFolder, User user, String n } public File copyFileToTemp(IFileAsset file, File tempFolder) throws IOException{ + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try (var lease = APILocator.getBinaryAssetStorageAPI().acquireCacheLease()) { + final File binary = ((Contentlet) file).getBinary(FileAssetAPI.BINARY_FIELD); + final File destination = new File(tempFolder, binary.getName()); + try (InputStream input = ((Contentlet) file).getBinaryStream(FileAssetAPI.BINARY_FIELD)) { + writeCompletedTempFile(destination, input); + } + return destination; + } + } File f = null; f = ((Contentlet)file).getBinary(FileAssetAPI.BINARY_FIELD); @@ -576,7 +637,8 @@ public File copyFileToTemp(IFileAsset file, File tempFolder) throws IOException{ } public File createTempFile(String path) throws IOException{ - File file = new File(getTempDir().getPath() + path); + File file = new File(getTempDir().getPath() + temporaryPath(path)); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) return file; String p = file.getPath().substring(0,file.getPath().lastIndexOf(File.separator)); File f = new File(p); f.mkdirs(); @@ -585,6 +647,20 @@ public File createTempFile(String path) throws IOException{ } public void copyTempDirToStorage(File fromFileFolder, String destPath, User user,boolean autoPublish) throws Exception{ + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final var staging = com.dotcms.storage.WebdavTemporaryStorage.getInstance(); + final var entry = staging.stat(fromFileFolder); + if (entry == null || !entry.directory()) throw new IOException("Missing WebDAV staging directory"); + destPath = stripMapping(destPath); + if (!destPath.endsWith("/")) destPath += "/"; + if (createFolder(destPath, user) == null) throw new IOException("Unable to create WebDAV destination folder"); + for (var child : staging.children(fromFileFolder)) { + final File file = staging.file(child.path()); + if (child.directory()) copyTempDirToStorage(file, destPath + file.getName(), user, autoPublish); + else copyTempFileToStorage(file, destPath + file.getName(), user, autoPublish); + } + return; + } if(fromFileFolder == null || !fromFileFolder.isDirectory()){ throw new IOException("The temp source file must be a directory"); } @@ -607,11 +683,26 @@ public void copyTempFileToStorage(File fromFile, String destPath,User user,boole if(fromFile == null){ throw new IOException("The temp source file must exist"); } + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final var staging = com.dotcms.storage.WebdavTemporaryStorage.getInstance(); + final var entry = staging.stat(fromFile); + try (InputStream input = Files.newInputStream(staging.materialize(entry).toPath())) { + setResourceContent(destPath, input, null, null, new Date(entry.modified()), user, autoPublish); + } + return; + } InputStream in = Files.newInputStream(fromFile.toPath()); setResourceContent(destPath, in, null, null, new Date(fromFile.lastModified()),user, autoPublish); } public void copyResource(String fromPath, String toPath, User user, boolean autoPublish) throws Exception { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try (InputStream input = getResourceContent(fromPath, user)) { + if (input == null) throw new IOException("Missing WebDAV copy source"); + setResourceContent(toPath, input, null, null, new Date(), user, autoPublish); + } + return; + } setResourceContent(toPath, getResourceContent(fromPath,user), null, null, user); } @@ -629,7 +720,11 @@ public void copyFolder(String sourcePath, String destinationPath, User user, boo if (!children[i].isFolder()) { - setResourceContent(destinationPath + "/" + children[i].getName(), getResourceContent(sourcePath + "/" + children[i].getName(),user), null, null, user); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + copyResource(sourcePath + "/" + children[i].getName(), destinationPath + "/" + children[i].getName(), user, autoPublish); + } else { + setResourceContent(destinationPath + "/" + children[i].getName(), getResourceContent(sourcePath + "/" + children[i].getName(),user), null, null, user); + } // ### Copy the permission ### // Source @@ -829,6 +924,13 @@ public void setResourceContent(String resourceUri, // Make sure we are not trying to delete the current version if (!contentlet.isLive() && !contentlet.isWorking()) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + if (storedBinaryLength(contentlet) == 0) { + conAPI.deleteVersion(contentlet, user, false); + break; + } + continue; + } final File binary = contentlet.getBinary(FileAssetAPI.BINARY_FIELD); @@ -843,9 +945,19 @@ public void setResourceContent(String resourceUri, } } } + } else if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + throw new IOException("WebDAV destination site or folder does not exist"); } } // setResourceContent. + static long storedBinaryLength(final Contentlet contentlet) throws IOException { + try (var lease = APILocator.getBinaryAssetStorageAPI().acquireCacheLease()) { + final File binary = contentlet.getBinary(FileAssetAPI.BINARY_FIELD); + // Missing storage is not evidence that a historical version was an empty Finder upload. + return binary == null ? -1 : Files.size(binary.toPath()); + } + } + private void validatePermissions(User user, boolean isAutoPub, boolean disableWorkflow, Contentlet fileAsset) throws DotDataException, DotSecurityException { if (isAutoPub && !perAPI.doesUserHavePermission(fileAsset, PermissionAPI.PERMISSION_PUBLISH, user)) { @@ -1006,7 +1118,17 @@ private Contentlet runCheckinPublishNoWorkflow(final String resourceUri, final U * @param fileName * @return created file */ - private File createFileInTemporalFolder(Field fieldVar, final String userId, final String fileName) { + File createFileInTemporalFolder(Field fieldVar, final String userId, final String fileName) throws IOException { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + if (fileName == null || fileName.isBlank() || fileName.equals(".") || fileName.equals("..") + || fileName.contains("/") || fileName.contains("\\") || fileName.indexOf('\0') >= 0) { + throw new IOException("Invalid WebDAV upload filename"); + } + final var root = java.nio.file.Path.of(APILocator.getFileAssetAPI().getRealAssetPathTmpBinary()); + Files.createDirectories(root); + // A request owns its staging bytes until check-in has stored them in S3. + return Files.createTempDirectory(root, "webdav-input-").resolve(fileName).toFile(); + } final String folderPath = new StringBuilder() .append(APILocator.getFileAssetAPI().getRealAssetPathTmpBinary()) .append(File.separator).append(userId).append(File.separator) @@ -1027,6 +1149,39 @@ private File createFileInTemporalFolder(Field fieldVar, final String userId, fin return fileData; } + /** Caller owns the input. Failed uploads leave the previous complete staging file intact. */ + static File writeCompletedTempFile(final File destination, final InputStream input) throws IOException { + final var configuredRoot = java.nio.file.Path.of(ConfigUtils.getAssetTempPath()).toAbsolutePath().normalize(); + final var root = configuredRoot.toFile().getCanonicalFile().toPath(); + final var target = destination.getCanonicalFile().toPath(); + if (!target.startsWith(root) || target.equals(root)) { + throw new IOException("WebDAV staging path is outside the temporary directory"); + } + final var supplied = destination.toPath().toAbsolutePath().normalize(); + final var boundary = supplied.startsWith(configuredRoot) ? configuredRoot : root; + if (!supplied.startsWith(boundary)) throw new IOException("Invalid WebDAV staging path"); + for (var path = supplied; !path.equals(boundary); path = path.getParent()) { + if (Files.isSymbolicLink(path)) throw new IOException("WebDAV staging path contains a symbolic link"); + } + Files.createDirectories(target.getParent()); + final var stage = Files.createTempFile(root, ".webdav-write-", ".tmp"); + try { + Files.copy(input, stage, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().store(destination, stage.toFile()); + } catch (DotDataException failure) { + throw new IOException("Unable to store WebDAV staging file", failure); + } + } + Files.move(stage, target, java.nio.file.StandardCopyOption.ATOMIC_MOVE, + java.nio.file.StandardCopyOption.REPLACE_EXISTING); + return destination; + } finally { + Files.deleteIfExists(stage); + } + } + public Folder createFolder(String folderUri, User user) throws IOException, DotDataException { Folder folder = null; folderUri = stripMapping(folderUri); @@ -1146,7 +1301,14 @@ public void move(String fromPath, String toPath, User user,boolean autoPublish)t final File tmpDir = new File(APILocator.getFileAssetAPI().getRealAssetPathTmpBinary() + File.separator+UUIDGenerator.generateUuid()); final File tmp = new File(tmpDir, toContentlet.getBinary(FileAssetAPI.BINARY_FIELD).getName()); - FileUtil.copyFile(origin.getBinary(FileAssetAPI.BINARY_FIELD), tmp); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try (InputStream input = origin.getBinaryStream(FileAssetAPI.BINARY_FIELD)) { + Files.createDirectories(tmpDir.toPath()); + Files.copy(input, tmp.toPath(), java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } + } else { + FileUtil.copyFile(origin.getBinary(FileAssetAPI.BINARY_FIELD), tmp); + } newVersion.setBinary(FileAssetAPI.BINARY_FIELD, tmp); newVersion.setLanguageId(defaultLang); @@ -1343,8 +1505,10 @@ public void removeObject(String uri, User user) throws IOException, DotDataExcep //Webdav calls the delete method when is creating a new file. But it creates the file with 0 content length. //No need to wait 10 seconds with files with 0 length. if(canDelete - || (fileAssetCont.getBinary(FileAssetAPI.BINARY_FIELD) != null - && fileAssetCont.getBinary(FileAssetAPI.BINARY_FIELD).length() <= 0)){ + || (com.dotcms.storage.AssetStorageFeature.isEnabled() + ? storedBinaryLength(fileAssetCont) == 0 + : (fileAssetCont.getBinary(FileAssetAPI.BINARY_FIELD) != null + && fileAssetCont.getBinary(FileAssetAPI.BINARY_FIELD).length() <= 0))){ try{ conAPI.archive(fileAssetCont, user, false); @@ -1777,8 +1941,10 @@ private Summary[] getChildrenData(String folderUriAux, User user) throws IOExcep Identifier identifier = APILocator.getIdentifierAPI().find(file); if(identifier!=null && identifier.getAssetType().equals("contentlet")){ fileUri = identifier.getPath(); - workingFile = ((Contentlet)file).getBinary(FileAssetAPI.BINARY_FIELD); - is = Files.newInputStream(workingFile.toPath()); + if (!com.dotcms.storage.AssetStorageFeature.isEnabled()) { + workingFile = ((Contentlet)file).getBinary(FileAssetAPI.BINARY_FIELD); + is = Files.newInputStream(workingFile.toPath()); + } idate = file.getModDate(); } @@ -1793,7 +1959,13 @@ private Summary[] getChildrenData(String folderUriAux, User user) throws IOExcep s.setCreateDate(idate); s.setModifyDate(fa.getModDate()); - s.setLength(is.available()); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final long length = storedBinaryLength((Contentlet) file); + if (length < 0) throw new IOException("Missing WebDAV asset"); + s.setLength(length); + } else { + s.setLength(is.available()); + } s.setHost(host); s.setFile(fa); returnValue.add(s); @@ -1804,6 +1976,9 @@ private Summary[] getChildrenData(String folderUriAux, User user) throws IOExcep } } } catch (Exception ex) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + throw new IOException("Unable to enumerate WebDAV folder assets", ex); + } Logger.debug(this, ex.toString()); } return returnValue.toArray(new Summary[returnValue.size()]); @@ -1832,6 +2007,9 @@ private InputStream getResourceContent(String resourceUri, User user) throws Exc Identifier identifier = APILocator.getIdentifierAPI().find(host, path); if(identifier!=null && identifier.getAssetType().equals("contentlet")){ Contentlet cont = conAPI.findContentletByIdentifier(identifier.getId(), false, defaultLang, user, false); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + return cont.getBinaryStream(FileAssetAPI.BINARY_FIELD); + } File workingFile = cont.getBinary(FileAssetAPI.BINARY_FIELD); is = Files.newInputStream(workingFile.toPath()); } diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/FileResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/FileResourceImpl.java index a00263860943..76f2d54226be 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/FileResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/FileResourceImpl.java @@ -111,6 +111,15 @@ public String checkRedirect(Request req) { } public Long getContentLength() { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + final long length = DotWebdavHelper.storedBinaryLength((Contentlet) file); + if (length < 0) throw new IOException("Missing WebDAV asset"); + return length; + } catch (IOException e) { + throw new DotRuntimeException("Unable to read WebDAV asset length", e); + } + } java.io.File workingFile; try { workingFile = ((Contentlet)file).getBinary(FileAssetAPI.BINARY_FIELD); @@ -156,6 +165,12 @@ public Long getMaxAgeSeconds() { } public void sendContent(OutputStream out, Range arg1, Map arg2, String arg3) throws IOException { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try (InputStream input = ((Contentlet) file).getBinaryStream(FileAssetAPI.BINARY_FIELD)) { + input.transferTo(out); + } + return; + } java.io.File f; try { f = ((Contentlet)file).getBinary(FileAssetAPI.BINARY_FIELD); diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/FolderResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/FolderResourceImpl.java index c9beeb8b8969..3928467a102d 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/FolderResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/FolderResourceImpl.java @@ -78,7 +78,10 @@ public CollectionResource createCollection(String newName) throws DotRuntimeExce final String hostFolderPath = new StringBuilder(File.separator).append(host.getHostname()) .append(!folderPath.endsWith(File.separator)?folderPath + File.separator : folderPath).toString(); - dotDavHelper.createTempFolder(hostFolderPath + newName); + final File created = dotDavHelper.createTempFolder(hostFolderPath + newName); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + return new TempFolderResourceImpl(path + (path.endsWith("/") ? "" : "/") + newName, created, isAutoPub); + } File file = new File(File.separator + host.getHostname() + folderPath); TempFolderResourceImpl tempFolderResource = new TempFolderResourceImpl(file.getPath(),file ,isAutoPub); return tempFolderResource; @@ -115,6 +118,11 @@ public Resource child(String childName) { throw new DotRuntimeException(e.getMessage(), e); } for (final Resource resource : children) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled() + && (resource instanceof TempFolderResourceImpl || resource instanceof TempFileResourceImpl)) { + if (resource.getName().equals(childName)) return resource; + continue; + } if(resource instanceof FolderResourceImpl){ final String name = ((FolderResourceImpl)resource).getFolder().getName(); if(name.equalsIgnoreCase(childName)){ @@ -407,4 +415,4 @@ public LockToken createAndLock(String name, LockTimeout timeout, LockInfo lockIn return lock(timeout, lockInfo).getLockToken(); } -} \ No newline at end of file +} diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java index 72f7cc78b99e..7bff0c1a9d10 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java @@ -117,6 +117,14 @@ public Resource child(String childName) { return null; } final User user = (User)HttpManager.request().getAuthorization().getTag(); + if (com.dotcms.storage.AssetStorageFeature.isEnabled() && dotDavHelper.isTempResource(childName)) { + try { + return dotDavHelper.temporaryChildren(path, isAutoPub).stream() + .filter(resource -> resource.getName().equals(childName)).findFirst().orElse(null); + } catch (IOException failure) { + throw new DotRuntimeException("Unable to find WebDAV staging child", failure); + } + } final String uri="/"+childName; try { @@ -201,7 +209,13 @@ public List getChildren() { prePath += "/"; } java.io.File tempDir = new java.io.File(dotDavHelper.getTempDir().getPath() + java.io.File.separator + host.getHostname()); - if(tempDir.exists() && tempDir.isDirectory()){ + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + frs.addAll(dotDavHelper.temporaryChildren(prePath + host.getHostname(), isAutoPub)); + } catch (IOException failure) { + throw new DotRuntimeException("Unable to list WebDAV staging resources", failure); + } + } else if(tempDir.exists() && tempDir.isDirectory()){ java.io.File[] files = tempDir.listFiles(); for (java.io.File file : files) { String tp = prePath + host.getHostname() + "/" + file.getName(); diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java index b580ba1f8cb7..c2ac9c632824 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java @@ -86,7 +86,10 @@ public LanguageFolderResourceImpl(String path) { */ public CollectionResource createCollection(String newName) { if(dotDavHelper.isTempResource(newName) && isLanguageRoot){ - dotDavHelper.createTempFolder(File.separator + "system" + File.separator + "languages" + File.separator + newName); + final File created = dotDavHelper.createTempFolder(File.separator + "system" + File.separator + "languages" + File.separator + newName); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + return new TempFolderResourceImpl(created.getPath(), created, true); + } File f = new File(File.separator + "system" + File.separator + "languages"); TempFolderResourceImpl tr = new TempFolderResourceImpl(f.getPath(),f ,true); return tr; @@ -101,6 +104,11 @@ public CollectionResource createCollection(String newName) { public Resource child(String childName) { List children = getChildren(); for (Resource resource : children) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled() + && (resource instanceof TempFolderResourceImpl || resource instanceof TempFileResourceImpl)) { + if (resource.getName().equals(childName)) return resource; + continue; + } if(resource instanceof LanguageFolderResourceImpl){ String name = ((LanguageFolderResourceImpl)resource).getFolder().getName(); if(name.equalsIgnoreCase(childName)){ @@ -121,6 +129,10 @@ public Resource child(String childName) { */ public List getChildren() { File[] children = folder.listFiles(); + if (children == null && com.dotcms.storage.AssetStorageFeature.isEnabled()) { + if (folder.exists()) throw new com.dotmarketing.exception.DotRuntimeException("Unable to list language directory " + folder); + children = new File[0]; + } List result = new ArrayList<>(); for (File file : children) { if(file.getName().endsWith(".properties") || file.getName().endsWith(".native") || file.getName().equals("archived") || folder.getName().equals("archived")){ @@ -134,6 +146,16 @@ public List getChildren() { } } File tempDir = dotDavHelper.getTempDir(); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final String prefix = Config.getBooleanProperty("WEBDAV_LEGACY_PATHING", false) + ? "/webdav/autopub/" : "/webdav/live/" + dotDavHelper.getLanguage() + "/"; + try { + result.addAll(dotDavHelper.temporaryChildren(prefix + "system/languages/" + path, true)); + return result; + } catch (IOException failure) { + throw new com.dotmarketing.exception.DotRuntimeException("Unable to list WebDAV staging resources", failure); + } + } File f = new File(tempDir.getPath() + File.separator + "system" + File.separator + "languages" + path); File[] c = f.listFiles(); if(c != null){ diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java index 3c9f1a0037ae..9d2b289ff0d2 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java @@ -47,7 +47,7 @@ public Resource getResource(String davHost, String url) { } public static Resource getResource(final String davHost, String url, final DotWebdavHelper dotDavHelper, final HostAPI hostAPI) { - url = url.toLowerCase(); + if (!com.dotcms.storage.AssetStorageFeature.isEnabled() || !dotDavHelper.isTempResource(url)) url = url.toLowerCase(); Logger.debug(ResourceFactoryImpl.class, "WebDav ResourceFactory: Host is " + davHost + " and the url is " + url); try{ dotDavHelper.stripMapping(url);//method also sets the language @@ -115,6 +115,12 @@ public static Resource getResource(final String davHost, String url, final DotWe // handle crappy dav clients temp files if(dotDavHelper.isTempResource(url)){ java.io.File tempFile = dotDavHelper.loadTempFile(url); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final var entry = com.dotcms.storage.WebdavTemporaryStorage.getInstance().stat(tempFile); + if (entry == null) return null; + return entry.directory() ? new TempFolderResourceImpl(url, tempFile, dotDavHelper.isAutoPub(url)) + : new TempFileResourceImpl(tempFile, url, dotDavHelper.isAutoPub(url)); + } if(tempFile == null || !tempFile.exists()){ return null; }else if(tempFile.isDirectory()){ @@ -204,6 +210,9 @@ public static Resource getResource(final String davHost, String url, final DotWe return fr; } } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled() && dotDavHelper.isTempResource(url)) { + throw new com.dotmarketing.exception.DotRuntimeException("Unable to resolve WebDAV staging resource", e); + } Logger.error(ResourceFactoryImpl.class, e.getMessage(), e); return null; } diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java index 9d3224f1d94a..e97fd2618b01 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java @@ -38,11 +38,17 @@ public class TempFileResourceImpl implements FileResource, LockableResource { private DotWebdavHelper dotDavHelper; private final File file; private final String url; + private final com.dotcms.storage.WebdavTemporaryStorage.Entry entry; private boolean isAutoPub = false; private PermissionAPI perAPI; public TempFileResourceImpl(File file, String url, boolean isAutoPub) { - if( file.isDirectory() ){ + entry = com.dotcms.storage.AssetStorageFeature.isEnabled() + ? com.dotcms.storage.WebdavTemporaryStorage.getInstance().stat(file) : null; + if (com.dotcms.storage.AssetStorageFeature.isEnabled() && entry == null) { + throw new IllegalArgumentException("Missing WebDAV staging file: " + url); + } + if( entry != null ? entry.directory() : file.isDirectory() ){ Logger.error(this, "Trying to get a temp file which is actually a directory!!!"); throw new IllegalArgumentException("Static resource must be a file, this is a directory: " + file.getAbsolutePath()); } @@ -55,7 +61,7 @@ public TempFileResourceImpl(File file, String url, boolean isAutoPub) { public String getUniqueId() { - return file.hashCode() + ""; + return entry == null ? file.hashCode() + "" : "webdav-temporary:" + entry.path(); } @@ -70,7 +76,7 @@ public int compareTo(Object o) { public void sendContent(OutputStream out, Range range, Map params, String arg3) throws IOException { - try(InputStream fis = Files.newInputStream(file.toPath())){ + try(InputStream fis = Files.newInputStream(getFile().toPath())){ BufferedInputStream bin = new BufferedInputStream(fis); final byte[] buffer = new byte[ 1024 ]; int n = 0; @@ -111,14 +117,14 @@ public String getRealm() { public Date getModifiedDate() { - Date dt = new Date(file.lastModified()); + Date dt = new Date(entry == null ? file.lastModified() : entry.modified()); // log.debug("static resource modified: " + dt); return dt; } public Long getContentLength() { - return file.length(); + return entry == null ? file.length() : entry.size(); } @@ -152,8 +158,15 @@ public void copyTo(CollectionResource collRes, String name) { if(collRes instanceof TempFolderResourceImpl){ TempFolderResourceImpl tr = (TempFolderResourceImpl)collRes; try { - FileUtil.copyFile(file, new File(tr.getFolder().getPath() + File.separator + name)); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final File destination = new File(tr.getFolder(), name); + if (destination.getCanonicalFile().equals(file.getCanonicalFile())) return; + com.dotcms.storage.WebdavTemporaryStorage.getInstance().copy(file, destination); + } else { + FileUtil.copyFile(file, new File(tr.getFolder().getPath() + File.separator + name)); + } } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); return; } @@ -165,6 +178,7 @@ public void copyTo(CollectionResource collRes, String name) { p = p + "/"; dotDavHelper.copyTempFileToStorage(file, p + name, user, isAutoPub); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); } }else if(collRes instanceof LanguageFolderResourceImpl){ @@ -177,8 +191,9 @@ public void copyTo(CollectionResource collRes, String name) { folder.mkdirs(); FileUtil.copyFile(f, new File(folder.getPath() + File.separator + new Date().toString())); } - FileUtil.copyFile(file, f); + FileUtil.copyFile(getFile(), f); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); return; } @@ -187,6 +202,14 @@ public void copyTo(CollectionResource collRes, String name) { public void delete() { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().delete(file); + return; + } catch (Exception failure) { + throw new com.dotmarketing.exception.DotRuntimeException("Unable to delete WebDAV staging file", failure); + } + } file.delete(); } @@ -196,9 +219,16 @@ public void moveTo(CollectionResource collRes, String name) { if(collRes instanceof TempFolderResourceImpl){ TempFolderResourceImpl tr = (TempFolderResourceImpl)collRes; try { - FileUtil.copyFile(file, new File(tr.getFolder().getPath() + File.separator + name)); - file.delete(); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final File destination = new File(tr.getFolder(), name); + if (destination.getCanonicalFile().equals(file.getCanonicalFile())) return; + com.dotcms.storage.WebdavTemporaryStorage.getInstance().copy(file, destination); + } else { + FileUtil.copyFile(file, new File(tr.getFolder().getPath() + File.separator + name)); + } + delete(); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); return; } @@ -209,8 +239,9 @@ public void moveTo(CollectionResource collRes, String name) { if(!p.endsWith("/")) p = p + "/"; dotDavHelper.copyTempFileToStorage(file, p + name, user, isAutoPub); - file.delete(); + delete(); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); } }else if(collRes instanceof LanguageFolderResourceImpl){ @@ -223,9 +254,10 @@ public void moveTo(CollectionResource collRes, String name) { folder.mkdirs(); FileUtil.copyFile(f, new File(folder.getPath() + File.separator + new Date().toString() + f.getName())); } - FileUtil.copyFile(file, f); - file.delete(); + FileUtil.copyFile(getFile(), f); + delete(); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); return; } @@ -245,7 +277,16 @@ public Date getCreateDate() { } + File getLogicalFile() { return file; } + public File getFile() { + if (entry != null) { + try { + return com.dotcms.storage.WebdavTemporaryStorage.getInstance().materialize(entry); + } catch (Exception failure) { + throw new com.dotmarketing.exception.DotRuntimeException("Unable to read WebDAV staging file", failure); + } + } return file; } diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java index a6ce67a1c76a..c7588dddf9a0 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java @@ -66,6 +66,10 @@ public CollectionResource createCollection(String newName) { public Resource child(String childName) { List children = getChildren(); for (Resource resource : children) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + if (resource.getName().equals(childName)) return resource; + continue; + } if(resource instanceof TempFolderResourceImpl){ String name = ((TempFolderResourceImpl)resource).getFolder().getName(); if(name.equalsIgnoreCase(childName)){ @@ -85,6 +89,20 @@ public Resource child(String childName) { * @see com.bradmcevoy.http.CollectionResource#getChildren() */ public List getChildren() { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + final var storage = com.dotcms.storage.WebdavTemporaryStorage.getInstance(); + final List result = new ArrayList<>(); + for (var entry : storage.children(folder)) { + final File file = storage.file(entry.path()); + result.add(entry.directory() ? new TempFolderResourceImpl(file.getPath(), file, isAutoPub) + : new TempFileResourceImpl(file, file.getPath(), isAutoPub)); + } + return result; + } catch (Exception failure) { + throw new com.dotmarketing.exception.DotRuntimeException("Unable to list WebDAV staging directory", failure); + } + } File[] children = folder.listFiles(); List result = new ArrayList<>(); for (File file : children) { @@ -147,6 +165,10 @@ public String getContentType(String arg0) { * @see com.bradmcevoy.http.Resource#getModifiedDate() */ public Date getModifiedDate() { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + final var entry = com.dotcms.storage.WebdavTemporaryStorage.getInstance().stat(folder); + return entry == null ? null : new Date(entry.modified()); + } return new Date(folder.lastModified()); } @@ -161,7 +183,11 @@ public String getRealm() { * @see com.bradmcevoy.http.Resource#getUniqueId() */ public String getUniqueId() { - return folder.hashCode() + ""; + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { return "webdav-temporary:" + com.dotcms.storage.WebdavTemporaryStorage.getInstance().key(folder); } + catch (IOException failure) { throw new com.dotmarketing.exception.DotRuntimeException(failure); } + } + return folder.hashCode() + ""; } /* (non-Javadoc) @@ -169,6 +195,10 @@ public String getUniqueId() { */ public Resource createNew(String newName, InputStream in, Long length, String contentType) throws IOException { File f = new File(folder.getPath() + File.separator + newName); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + DotWebdavHelper.writeCompletedTempFile(f, in); + return new TempFileResourceImpl(f, f.getPath(), isAutoPub); + } if(!f.exists()){ String p = f.getPath().substring(0,f.getPath().lastIndexOf(File.separator)); File fe = new File(p); @@ -196,8 +226,15 @@ public void copyTo(CollectionResource collRes, String name) { try { String p = tr.getFolder().getPath(); File dest = new File(tr.getFolder().getPath() + File.separator + name); - FileUtil.copyDirectory(folder, dest); + if (com.dotcms.storage.AssetStorageFeature.isEnabled() + && dest.getCanonicalFile().equals(folder.getCanonicalFile())) return; + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().copy(folder, dest); + } else { + FileUtil.copyDirectory(folder, dest); + } } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); return; } @@ -209,6 +246,7 @@ public void copyTo(CollectionResource collRes, String name) { try { dotDavHelper.copyTempDirToStorage(folder, p + name, user,isAutoPub); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); } }else if(collRes instanceof LanguageFolderResourceImpl){ @@ -220,6 +258,14 @@ public void copyTo(CollectionResource collRes, String name) { * @see com.bradmcevoy.http.DeletableResource#delete() */ public void delete() { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + try { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().delete(folder); + return; + } catch (Exception failure) { + throw new com.dotmarketing.exception.DotRuntimeException("Unable to delete WebDAV staging directory", failure); + } + } folder.delete(); } @@ -246,9 +292,17 @@ public void moveTo(CollectionResource collRes, String name) { TempFolderResourceImpl tr = (TempFolderResourceImpl)collRes; try { File dest = new File(tr.getFolder().getPath() + File.separator + name); - FileUtil.copyDirectory(folder, dest); - FileUtil.deltree(folder, true); + if (com.dotcms.storage.AssetStorageFeature.isEnabled() + && dest.getCanonicalFile().equals(folder.getCanonicalFile())) return; + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().copy(folder, dest); + } else { + FileUtil.copyDirectory(folder, dest); + } + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) delete(); + else FileUtil.deltree(folder, true); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); return; } @@ -259,16 +313,20 @@ public void moveTo(CollectionResource collRes, String name) { p = p + "/"; try { dotDavHelper.copyTempDirToStorage(folder, p + name, user, isAutoPub); - FileUtil.deltree(folder, true); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) delete(); + else FileUtil.deltree(folder, true); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); } }else if(collRes instanceof HostResourceImpl){ HostResourceImpl hr = (HostResourceImpl)collRes; try { dotDavHelper.copyTempDirToStorage(folder, "/" + hr.getHost().getHostname() + "/"+ name, user, isAutoPub); - FileUtil.deltree(folder, true); + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) delete(); + else FileUtil.deltree(folder, true); } catch (Exception e) { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) throw new com.dotmarketing.exception.DotRuntimeException(e); Logger.error(this, e.getMessage(), e); } }else if(collRes instanceof LanguageFolderResourceImpl){ @@ -280,6 +338,7 @@ public void moveTo(CollectionResource collRes, String name) { * @see com.bradmcevoy.http.PropFindableResource#getCreateDate() */ public Date getCreateDate() { + if (com.dotcms.storage.AssetStorageFeature.isEnabled()) return getModifiedDate(); Date dt = new Date(folder.lastModified()); // log.debug("static resource modified: " + dt); return dt; diff --git a/dotCMS/src/test/java/com/dotcms/storage/AssetStorageFeatureTest.java b/dotCMS/src/test/java/com/dotcms/storage/AssetStorageFeatureTest.java index 8c24e9068cb7..aa404f883e0d 100644 --- a/dotCMS/src/test/java/com/dotcms/storage/AssetStorageFeatureTest.java +++ b/dotCMS/src/test/java/com/dotcms/storage/AssetStorageFeatureTest.java @@ -125,6 +125,32 @@ void databaseQueryFailuresAreNotReportedAsMissingObjectsWhenEnabled() throws Exc } } + @Test + void metadataWriteFailurePropagatesOnlyWhenEnabled() throws Exception { + var storage = mock(FileStorageAPI.class); + doThrow(new DotDataException("S3 metadata upload failed")) + .when(storage).putCustomMetadataAttributes(any(), any()); + when(storage.setMetadata(any(), any())).thenThrow(new DotDataException("S3 metadata upload failed")); + var content = new com.dotmarketing.portlets.contentlet.model.Contentlet(); + content.setInode("abc123"); + content.getMap().put("HeroImage", root.resolve("File.PNG").toFile()); + try (var locator = mockStatic(com.dotmarketing.business.APILocator.class); + var caches = mockStatic(com.dotmarketing.business.CacheLocator.class)) { + locator.when(com.dotmarketing.business.APILocator::getFileStorageAPI).thenReturn(storage); + caches.when(com.dotmarketing.business.CacheLocator::getMetadataCache) + .thenReturn(mock(com.dotmarketing.portlets.contentlet.business.MetadataCache.class)); + var api = new FileMetadataAPIImpl(); + Map> attributes = Map.of("HeroImage", Map.of("credit", "Author")); + assertThrows(DotDataException.class, () -> api.putCustomMetadataAttributesForCheckin(content, attributes)); + assertThrows(DotDataException.class, () -> api.putCustomMetadataAttributes("temp_upload", attributes)); + Config.setProperty(AssetStorageFeature.FLAG, false); + assertDoesNotThrow(() -> api.putCustomMetadataAttributesForCheckin(content, attributes)); + assertDoesNotThrow(() -> api.putCustomMetadataAttributes("temp_upload", attributes)); + verify(storage, times(3)).putCustomMetadataAttributes(any(), any()); + verify(storage).setMetadata(any(), any()); + } + } + @Test void filesystemMetadataReplacementKeepsPriorValueDuringAndAfterFailedSerialization() throws Exception { final var fs = filesystem(); diff --git a/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java b/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java index fe41befa58a2..1f037b7870b7 100644 --- a/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java +++ b/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java @@ -169,6 +169,34 @@ void installationsKeepIndependentOwnersWhileSharingBinaryBytesAndExtraction() th } finally { secondClient.shutdownTransferManager(); } } + @Test + void namespaceCoversHashedMetadataAndConditionalStagingRecords() throws Exception { + final var first = namespaced("Prod", AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode.SHA256, storage); + final var second = namespaced("prod", AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode.SHA256, storage); + final String path = "/a/b/abc123/HeroImage-metadata.json"; + first.pushObject("dotmetadata", path, new JsonWriterDelegate(), "first editorial value", Map.of()); + second.pushObject("dotmetadata", path, new JsonWriterDelegate(), "second editorial value", Map.of()); + assertEquals(List.of(path), first.listObjectPaths("dotmetadata", "/a/b/abc123/")); + assertEquals("first editorial value", first.pullObject("dotmetadata", path, new JsonReaderDelegate<>(String.class))); + first.deleteObjectAndReferences("dotmetadata", path); + assertEquals("second editorial value", second.pullObject("dotmetadata", path, new JsonReaderDelegate<>(String.class))); + + final var stagingA = namespaced("Prod", AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode.NONE, storage); + final var stagingB = namespaced("prod", AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode.NONE, storage); + final String record = "entries/Mixed-Case.json"; + final String firstVersion = stagingA.writeObjectIfMatch("webdav-temporary", record, "first upload", null); + final String secondVersion = stagingB.writeObjectIfMatch("webdav-temporary", record, "second upload", null); + assertNotNull(firstVersion); + assertNotNull(secondVersion, "Same logical path can be created independently in another namespace"); + assertNull(stagingB.writeObjectIfMatch("webdav-temporary", record, "stale edit", firstVersion)); + assertEquals(record, stagingA.listObjectSnapshots("webdav-temporary", "entries/").get(0).path()); + stagingA.deleteObjectReference("webdav-temporary", record); + assertEquals("second upload", stagingB.readObjectSnapshot("webdav-temporary", record, + new JsonReaderDelegate<>(String.class)).value()); + assertFalse(stagingA.existsObject("webdav-temporary", record)); + assertTrue(stagingB.existsObject("webdav-temporary", record)); + } + private AmazonS3StoragePersistenceAPIImpl namespaced(String namespace, AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode mode, com.dotcms.enterprise.publishing.storage.Storage backing) { @@ -186,6 +214,252 @@ private String readRemote(AmazonS3StoragePersistenceAPIImpl remote, String group finally { remote.releaseRetrievedFile(file); } } + @Test + void versionInventoryFailurePropagatesAsCheckedStorageFailure() { + final var failedStorage = spy(storage); + final var outage = new com.dotmarketing.exception.DotRuntimeException("S3 listing unavailable"); + doThrow(outage).when(failedStorage).listObjects(bucket, "webdav-temporary/data/"); + final var failed = new AmazonS3StoragePersistenceAPIImpl(failedStorage, bucket, + AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode.NONE); + final var failure = assertThrows(com.dotmarketing.exception.DotDataException.class, + () -> failed.listObjectSnapshots(WebdavTemporaryStorage.GROUP, "data/")); + assertSame(outage, failure.getCause()); + } + + @Test + void webdavStagingSurvivesColdNodesOverwriteCopyAndDelete() throws Exception { + final var remote = spy(s3); + final var first = new WebdavTemporaryStorage(remote, root.resolve("dav-a")); + final var second = new WebdavTemporaryStorage(s3, root.resolve("dav-b")); + final File directory = first.file("example.com/(Mixed-Staging)"); + final String path = "example.com/(Mixed-Staging)/Hero-Mixed.GIF"; + final File source = Files.writeString(root.resolve("completed-upload"), "first complete bytes").toFile(); + first.mkdir(directory); + first.mkdir(first.file("example.com/(Mixed-Staging)/Empty")); + first.store(first.file(path), source); + final var snapshot = second.stat(second.file(path)); + assertEquals("Hero-Mixed.GIF", second.materialize(snapshot).getName()); + assertEquals("first complete bytes", Files.readString(second.materialize(snapshot).toPath())); + assertFalse(second.file(path).exists(), "Logical lookup must not rely on local pathname files"); + assertEquals(2, second.children(second.file("example.com/(Mixed-Staging)")).size(), + "S3 entries: " + s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "entries/") + + "; keys: " + client.listObjectsV2(bucket).getObjectSummaries().stream().map(o -> o.getKey()).toList()); + assertNull(second.stat(second.file(path.toLowerCase(java.util.Locale.ROOT)))); + assertThrows(java.io.IOException.class, () -> first.store(directory, source)); + + Files.writeString(source.toPath(), "replacement bytes"); + doReturn(false).when(remote).backfillFile(anyString(), anyString(), any()); + assertThrows(java.io.IOException.class, () -> first.store(first.file(path), source)); + assertEquals(snapshot.dataKey(), second.stat(second.file(path)).dataKey()); + doCallRealMethod().when(remote).backfillFile(anyString(), anyString(), any()); + first.store(first.file(path), source); + assertNotEquals(snapshot.dataKey(), second.stat(second.file(path)).dataKey()); + Files.delete(second.materialize(snapshot).toPath()); + assertEquals("first complete bytes", Files.readString(second.materialize(snapshot).toPath())); + assertEquals("replacement bytes", Files.readString(second.materialize(second.stat(second.file(path))).toPath())); + + final File copied = second.file("example.com/(Copy)"); + second.copy(second.file("example.com/(Mixed-Staging)"), copied); + assertTrue(second.stat(new File(copied, "Empty")).directory()); + assertEquals("replacement bytes", Files.readString(second.materialize(second.stat(new File(copied, "Hero-Mixed.GIF"))).toPath())); + second.copy(copied, copied); + assertNotNull(second.stat(copied)); + assertThrows(java.io.IOException.class, () -> second.copy(copied, new File(copied, "Nested"))); + final File sibling = first.file("example.com/(Copy)-neighbor.TXT"); + first.store(sibling, source); + // Stale local bytes cannot resurrect a remotely deleted resource. + Files.createDirectories(copied.toPath()); + Files.writeString(copied.toPath().resolve("Hero-Mixed.GIF"), "stale local bytes"); + first.delete(first.file("example.com/(Copy)")); + assertNull(second.stat(new File(copied, "Hero-Mixed.GIF"))); + assertNull(second.stat(copied)); + assertNotNull(second.stat(second.file("example.com/(Copy)-neighbor.TXT"))); + first.delete(directory); + assertNull(second.stat(second.file(path))); + assertTrue(second.children(second.file("example.com/(Mixed-Staging)")).isEmpty()); + + Files.createDirectories(root.resolve("dav-a")); + Files.createSymbolicLink(root.resolve("dav-a/alias"), root.resolve("dav-a/elsewhere")); + assertThrows(java.io.IOException.class, () -> first.file("alias/file")); + assertThrows(java.io.IOException.class, () -> first.file("../escape")); + assertThrows(java.io.IOException.class, () -> first.file(".webdav-cache/data/fake")); + clearInvocations(remote); + Config.setProperty(AssetStorageFeature.FLAG, false); + assertThrows(com.dotmarketing.exception.DotDataException.class, () -> first.mkdir(directory)); + verifyNoInteractions(remote); + } + + @Test + void conditionalS3RecordsRejectStaleWritesAndTombstoneRaces() throws Exception { + final String group = WebdavTemporaryStorage.GROUP; + s3.createGroup(group); + final String path = "conditional/Record.json"; + final var first = new java.util.HashMap(Map.of("value", "first")); + final var second = new java.util.HashMap(Map.of("value", "second")); + final String version = s3.writeObjectIfMatch(group, path, first, null); + assertNotNull(version); + assertNull(s3.writeObjectIfMatch(group, path, second, null)); + assertNotNull(s3.writeObjectIfMatch(group, path, second, version)); + assertNull(s3.writeObjectIfMatch(group, path, first, version)); + final var tombstone = new java.util.HashMap(Map.of("deleted", UUID.randomUUID().toString())); + assertNull(s3.writeObjectIfMatch(group, path, tombstone, version)); + final var current = s3.readObjectSnapshot(group, path, new JsonReaderDelegate<>(Map.class)); + assertEquals(second, current.value()); + assertTrue(current.modified() > 0); + assertNotNull(s3.writeObjectIfMatch(group, path, tombstone, current.version())); + assertNull(s3.writeObjectIfMatch(group, path, first, current.version()), "A late writer cannot resurrect a tombstoned reservation"); + assertEquals(tombstone, s3.readObjectSnapshot(group, path, new JsonReaderDelegate<>(Map.class)).value()); + } + + @Test + void webdavCleanupRetainsReferencedOldPayloadThenExpiresTheRecordAndBytes() throws Exception { + final var delayed = spy(s3); + final var dav = new WebdavTemporaryStorage(delayed, root.resolve("dav")); + final File logical = dav.file("example.com/(Temporary)/Mixed%2E.GIF"); + final File source = Files.writeString(root.resolve("completed"), "referenced bytes").toFile(); + doAnswer(call -> { + final boolean uploaded = (boolean) call.callRealMethod(); + Thread.sleep(1100); // Make payload older than the subsequently published record at S3's timestamp precision. + return uploaded; + }).when(delayed).backfillFile(anyString(), anyString(), any()); + dav.store(logical, source); + final var current = dav.stat(logical); + final var cleanupTime = new java.util.concurrent.atomic.AtomicLong( + s3.listObjectSnapshots(WebdavTemporaryStorage.GROUP, "entries/").get(0).modified() / 1000 * 1000); + pinWebdavCleanupClock(delayed, cleanupTime); + cleanWebdavNow(dav); + assertEquals("referenced bytes", Files.readString(dav.materialize(current).toPath())); + assertNotNull(dav.stat(logical), "An aged payload remains protected by its newer completed record"); + Thread.sleep(1100); + cleanupTime.set(0); + cleanWebdavNow(dav); + assertNull(dav.stat(logical)); + assertWebdavTombstones(); + assertTrue(s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "data/").isEmpty()); + assertTrue(s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "maintenance/").isEmpty()); + } + + @Test + void webdavCleanupFencesAnExpiredUploadBeforeItCanPublish() throws Exception { + final var delayed = spy(s3); + final var writer = new WebdavTemporaryStorage(delayed, root.resolve("writer")); + final var cleaner = new WebdavTemporaryStorage(s3, root.resolve("cleaner")); + final File logical = writer.file("example.com/(Temporary)/Late.TXT"); + final File source = Files.writeString(root.resolve("late-upload"), "late bytes").toFile(); + doAnswer(call -> { + final boolean uploaded = (boolean) call.callRealMethod(); + Thread.sleep(1100); + cleanWebdavNow(cleaner); + return uploaded; + }).when(delayed).backfillFile(anyString(), anyString(), any()); + assertThrows(com.dotmarketing.exception.DotDataException.class, () -> writer.store(logical, source)); + assertNull(writer.stat(logical)); + assertEquals("late bytes", Files.readString(source.toPath()), "A rejected publication retains the caller's source"); + assertWebdavTombstones(); + assertTrue(s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "data/").isEmpty()); + } + + @Test + void webdavCleanupAbortsIfAWriterReplacesTheInspectedRecord() throws Exception { + final var cleanupRemote = spy(s3); + final var writer = new WebdavTemporaryStorage(s3, root.resolve("writer")); + final var cleaner = new WebdavTemporaryStorage(cleanupRemote, root.resolve("cleaner")); + final File logical = writer.file("example.com/(Temporary)/Replaced.TXT"); + final File source = Files.writeString(root.resolve("before-cleanup"), "old bytes").toFile(); + writer.store(logical, source); + Thread.sleep(1100); + final var raced = new java.util.concurrent.atomic.AtomicBoolean(); + doAnswer(call -> { + final String path = call.getArgument(1); + if (path.startsWith("entries/") && raced.compareAndSet(false, true)) { + Files.writeString(source.toPath(), "new bytes"); + writer.store(logical, source); + } + return call.callRealMethod(); + }).when(cleanupRemote).writeObjectIfMatch(anyString(), anyString(), any(), anyString()); + assertThrows(com.dotmarketing.exception.DotDataException.class, () -> cleanWebdavNow(cleaner)); + assertTrue(raced.get()); + assertEquals("new bytes", Files.readString(writer.materialize(writer.stat(logical)).toPath())); + // The failed pass must not have collected any payloads using its stale reference inventory. + assertEquals(2, s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "data/").size()); + pinWebdavCleanupClock(cleanupRemote, new java.util.concurrent.atomic.AtomicLong( + s3.listObjectSnapshots(WebdavTemporaryStorage.GROUP, "entries/").get(0).modified() / 1000 * 1000)); + cleanWebdavNow(cleaner); + assertEquals("new bytes", Files.readString(writer.materialize(writer.stat(logical)).toPath())); + assertEquals(1, s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "data/").size()); + } + + private void assertWebdavTombstones() throws Exception { + for (String path : s3.listObjectPaths(WebdavTemporaryStorage.GROUP, "entries/")) { + final var value = (Map) s3.readObjectSnapshot(WebdavTemporaryStorage.GROUP, path, + new JsonReaderDelegate<>(Map.class)).value(); + assertFalse(value.containsKey("dataKey")); + assertFalse(value.containsKey("pending")); + assertTrue(value.containsKey("mutation"), "Retain the fencing token for delayed writers"); + } + } + + private static void pinWebdavCleanupClock(AmazonS3StoragePersistenceAPIImpl remote, + java.util.concurrent.atomic.AtomicLong time) throws Exception { + doAnswer(call -> { + final var actual = (AmazonS3StoragePersistenceAPIImpl.ObjectSnapshot) call.callRealMethod(); + if (actual != null && actual.path().startsWith("maintenance/") && time.get() != 0) { + return new AmazonS3StoragePersistenceAPIImpl.ObjectSnapshot(actual.path(), actual.value(), actual.version(), time.get()); + } + return actual; + }).when(remote).readObjectSnapshot(anyString(), anyString(), any()); + } + + private static void cleanWebdavNow(WebdavTemporaryStorage dav) throws Exception { + final String previous = Config.getStringProperty("CLEANUP_TMP_FILES_OLDER_THAN_HOURS", null); + try { + Config.setProperty("CLEANUP_TMP_FILES_OLDER_THAN_HOURS", 0); + dav.cleanupExpired(); + } finally { + Config.setProperty("CLEANUP_TMP_FILES_OLDER_THAN_HOURS", previous); + } + } + + @Test + void temporaryUploadsSurviveColdCacheAndExpireWithTheirAccessRecord() throws Exception { + TemporaryAssetStorageTest.assertRoundTrip(s3, root); + } + + @Test + void temporaryMetadataUsesPortableS3KeysAndBypassesNodeCaches() throws Exception { + final var remote = new AmazonS3StoragePersistenceAPIImpl(storage, bucket, + AmazonS3StoragePersistenceAPIImpl.PathEncryptionMode.SHA256); + final var providers = mock(StoragePersistenceProvider.class); + when(providers.getStorage(StorageType.S3)).thenReturn(remote); + when(providers.getStorage(StorageType.FILE_SYSTEM)).thenReturn(fs); + fs.addGroupMapping(FileMetadataAPI.DOT_METADATA, root.toFile()); + final var cache = mock(com.dotmarketing.portlets.contentlet.business.MetadataCache.class); + when(cache.getMetadataMap(anyString())).thenReturn(Map.of("dot:focalPoint", "stale")); + final var files = new FileStorageAPIImpl(new JsonReaderDelegate<>(Map.class), new JsonWriterDelegate(), + mock(MetadataGenerator.class), providers, cache); + try (var locator = mockStatic(com.dotmarketing.business.APILocator.class); + var caches = mockStatic(com.dotmarketing.business.CacheLocator.class)) { + locator.when(com.dotmarketing.business.APILocator::getFileStorageAPI).thenReturn(files); + caches.when(com.dotmarketing.business.CacheLocator::getMetadataCache).thenReturn(cache); + configUtils.when(ConfigUtils::getAssetTempPath).thenReturn(root.resolve("node-a/tmp_upload").toString()); + final var nodeA = new FileMetadataAPIImpl(); + final String id = "temp_Mixed-Case"; + nodeA.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of("credit", "Author", "focalPoint", "0.25,0.5"))); + configUtils.when(ConfigUtils::getAssetTempPath).thenReturn(root.resolve("node-b/other/tmp_upload").toString()); + final var nodeB = new FileMetadataAPIImpl(); + assertEquals("0.25,0.5", nodeB.getMetadata(id).orElseThrow().getCustomMeta().get("focalPoint")); + nodeB.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of("focalPoint", "0.75,0.5"))); + assertEquals(Map.of("credit", "Author", "focalPoint", "0.75,0.5"), + nodeA.getMetadata(id).orElseThrow().getCustomMeta()); + nodeB.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of())); + assertTrue(nodeA.getMetadata(id).orElseThrow().getCustomMeta().isEmpty()); + verify(cache, never()).getMetadataMap(anyString()); + remote.deleteObjectAndReferences(FileMetadataAPI.DOT_METADATA, + "/tmp_upload/" + id + "/" + id + FileMetadataAPI.META_TMP); + assertTrue(nodeA.getMetadata(id).isEmpty()); + } + } + @Test void publishingArchivesPreserveCaseAndLastCompletedUpload() throws Exception { final var archives = new com.dotcms.publishing.output.BundleArchiveStorage( diff --git a/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java b/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java new file mode 100644 index 000000000000..1ca248a6e204 --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java @@ -0,0 +1,143 @@ +package com.dotcms.storage; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.dotcms.storage.binary.BinaryAssetStorageAPI; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.util.Config; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TemporaryAssetStorageTest { + @TempDir Path root; + + /** Also called by the real MinIO test, using two independent local cache roots. */ + static void assertRoundTrip(final StoragePersistenceAPI remote, final Path root) throws Exception { + final String flag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + final String ttl = Config.getStringProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", null); + try (var locator = mockStatic(APILocator.class)) { + Config.setProperty(AssetStorageFeature.FLAG, true); + Config.setProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", 1800); + final var metadata = mock(FileStorageAPI.class); + final var binaries = mock(BinaryAssetStorageAPI.class); + locator.when(APILocator::getFileStorageAPI).thenReturn(metadata); + locator.when(APILocator::getBinaryAssetStorageAPI).thenReturn(binaries); + final String id = "temp_Mixed-Case"; + final var nodeA = new TemporaryAssetStorage(remote, root.resolve("node-a")); + final var nodeB = new TemporaryAssetStorage(remote, root.resolve("node-b")); + final File source = nodeA.file(id, "Nested/Hero.GIF"); + Files.createDirectories(source.toPath().getParent()); + Files.writeString(source.toPath(), "complete upload bytes"); + Files.writeString(source.toPath().resolveSibling(TemporaryAssetStorage.PERMISSIONS_FILE), "[\"owner\"]"); + nodeA.store(id, source); + final var receipt = nodeB.receipt(id).orElseThrow(); + assertTrue(nodeB.exists(receipt)); + assertTrue(nodeB.retrieve(receipt, List.of("stranger")).isEmpty()); + assertFalse(Files.exists(root.resolve("node-b")), "Denied access must not materialize a file"); + Files.delete(source.toPath()); + final File restored = nodeB.retrieve(receipt, List.of("owner")).orElseThrow(); + assertEquals("Nested/Hero.GIF", root.resolve("node-b").resolve(id).toRealPath().relativize(restored.toPath()).toString()); + assertEquals("complete upload bytes", Files.readString(restored.toPath())); + assertEquals(receipt.modifiedAt(), restored.lastModified(), "Restoration must not renew expiry"); + assertEquals("complete upload bytes", Files.readString(nodeA.retrieve(receipt, List.of("owner")).orElseThrow().toPath())); + Config.setProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", 0); + assertFalse(nodeB.exists(receipt)); + assertTrue(nodeB.retrieve(receipt, List.of("owner")).isEmpty()); + + doThrow(new DotDataException("metadata unavailable")).when(metadata).removeMetaData(any()); + assertThrows(DotDataException.class, nodeB::cleanupExpired); + assertTrue(remote.existsObject(TemporaryAssetStorage.GROUP, receipt.key()), "Failed cleanup must retain its retry record and payload"); + assertTrue(nodeB.receipt(id).isPresent()); + doReturn(true).when(metadata).removeMetaData(any()); + nodeB.cleanupExpired(); + assertFalse(remote.existsObject(TemporaryAssetStorage.GROUP, receipt.key())); + assertTrue(nodeB.receipt(id).isEmpty()); + assertFalse(restored.exists()); + verify(binaries).deleteGeneratedFiles(id); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, flag); + Config.setProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", ttl); + } + } + + @Test + void coldRoundTripAuthorizationAndExpiry() throws Exception { + final var remote = memoryRemote(); + assertRoundTrip(remote, root); + verify(remote, times(2)).pullFile(anyString(), anyString()); + verify(remote, times(2)).releaseRetrievedFile(any()); + } + + @Test + void rejectedUploadRetainsLocalBytesAndDisabledStorageDoesNoRemoteWork() throws Exception { + final String flag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + try { + final var remote = memoryRemote(); + final var store = new TemporaryAssetStorage(remote, root); + Config.setProperty(AssetStorageFeature.FLAG, true); + final String id = "temp_Failed"; + final File source = store.file(id, "Image.GIF"); + Files.createDirectories(source.toPath().getParent()); + Files.writeString(source.toPath(), "retain me"); + Files.writeString(source.toPath().resolveSibling(TemporaryAssetStorage.PERMISSIONS_FILE), "[\"owner\"]"); + doReturn(false).when(remote).backfillFile(anyString(), anyString(), any()); + assertThrows(DotDataException.class, () -> store.store(id, source)); + assertEquals("retain me", Files.readString(source.toPath())); + assertTrue(store.isManagedLocally(id)); + assertTrue(store.receipt(id).isPresent(), "Interrupted uploads must remain discoverable for expiry"); + assertFalse(store.exists(store.receipt(id).orElseThrow())); + for (String name : List.of("../other/file", "/outside", "whoCanUse.tmp", ".s3-upload", "x.meta.tmp", "a\\b")) { + assertThrows(IllegalArgumentException.class, () -> store.file(id, name)); + } + Files.createSymbolicLink(root.resolve(id).resolve("alias"), root.resolve("another")); + assertThrows(IllegalArgumentException.class, () -> store.file(id, "alias/file")); + clearInvocations(remote); + Config.setProperty(AssetStorageFeature.FLAG, false); + store.store(id, source); + store.cleanupExpired(); + assertTrue(store.receipt(id).isEmpty()); + assertTrue(store.retrieve(new TemporaryAssetStorage.Receipt(id, "Image.GIF", System.currentTimeMillis(), List.of("owner")), List.of("owner")).isEmpty()); + verifyNoInteractions(remote); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, flag); + } + } + + private StoragePersistenceAPI memoryRemote() throws Exception { + final var remote = mock(StoragePersistenceAPI.class); + final Map objects = new HashMap<>(); + when(remote.backfillObject(anyString(), anyString(), any(), any(), any())).thenAnswer(call -> { + final String key = call.getArgument(1); + final Object value = call.getArgument(4); + return objects.putIfAbsent(key, value) == null || objects.get(key).equals(value); + }); + when(remote.backfillFile(anyString(), anyString(), any())).thenAnswer(call -> { + final String key = call.getArgument(1); + final byte[] bytes = Files.readAllBytes(call.getArgument(2).toPath()); + return objects.putIfAbsent(key, bytes) == null || java.util.Arrays.equals((byte[]) objects.get(key), bytes); + }); + when(remote.pullObject(anyString(), anyString(), any())).thenAnswer(call -> objects.get(call.getArgument(1))); + when(remote.existsObject(anyString(), anyString())).thenAnswer(call -> objects.containsKey(call.getArgument(1))); + when(remote.pullFile(anyString(), anyString())).thenAnswer(call -> { + final byte[] bytes = (byte[]) objects.get(call.getArgument(1)); + return bytes == null ? null : Files.write(Files.createTempFile(root, "download-", ".tmp"), bytes).toFile(); + }); + doAnswer(call -> { Files.delete(call.getArgument(0).toPath()); return null; }).when(remote).releaseRetrievedFile(any()); + when(remote.listObjectPaths(anyString(), anyString())).thenAnswer(call -> + objects.keySet().stream().filter(key -> key.startsWith(call.getArgument(1))).toList()); + when(remote.deleteObjectAndReferences(anyString(), anyString())).thenAnswer(call -> { + final String prefix = call.getArgument(1); + objects.remove(prefix); + return true; + }); + return remote; + } +} diff --git a/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java b/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java new file mode 100644 index 000000000000..ed12701f23be --- /dev/null +++ b/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java @@ -0,0 +1,92 @@ +package com.dotcms.storage; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.dotmarketing.business.APILocator; +import com.dotmarketing.business.CacheLocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.portlets.contentlet.business.MetadataCache; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.ConfigUtils; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; + +class TemporaryMetadataStorageTest { + @Test + void portableMetadataPreservesLegacyValuesAndClearedFocalPointsAcrossNodes() throws Exception { + final String oldFlag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + final String id = "temp_Mixed-Case"; + final Map legacy = Map.of("dot:credit", "Author", "dot:focalPoint", "0.25,0.5"); + final var shared = new AtomicReference>(); + final var files = mock(FileStorageAPI.class); + try (var locator = mockStatic(APILocator.class); var caches = mockStatic(CacheLocator.class); + var paths = mockStatic(ConfigUtils.class)) { + Config.setProperty(AssetStorageFeature.FLAG, true); + locator.when(APILocator::getFileStorageAPI).thenReturn(files); + caches.when(CacheLocator::getMetadataCache).thenReturn(mock(MetadataCache.class)); + paths.when(ConfigUtils::getAssetTempPath).thenReturn("/node-a/assets/tmp_upload"); + when(files.retrieveRawMetaData(any())).thenAnswer(call -> { + StorageKey key = call.getArgument(0); + if (key.getStorage() == StorageType.S3) { + assertEquals("/tmp_upload/" + id + "/" + id + FileMetadataAPI.META_TMP, key.getPath()); + return shared.get(); + } + assertEquals(StorageType.FILE_SYSTEM, key.getStorage()); + assertTrue(key.getPath().startsWith("/node-a/")); + return legacy; + }); + when(files.setMetadata(any(), any())).thenAnswer(call -> { + FetchMetadataParams request = call.getArgument(0); + assertEquals(StorageType.S3, request.getStorageKey().getStorage()); + assertFalse(request.isCache()); + shared.set(new HashMap<>(call.>getArgument(1))); + return true; + }); + final var nodeA = new FileMetadataAPIImpl(); + nodeA.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of("focalPoint", "0.75,0.5"))); + assertEquals("0.25,0.5", legacy.get("dot:focalPoint"), "Legacy metadata must remain unchanged"); + paths.when(ConfigUtils::getAssetTempPath).thenReturn("/node-b/different/tmp_upload"); + final var nodeB = new FileMetadataAPIImpl(); + assertEquals(Map.of("credit", "Author", "focalPoint", "0.75,0.5"), + nodeB.getMetadata(id).orElseThrow().getCustomMeta()); + nodeB.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of())); + assertTrue(nodeA.getMetadata(id).orElseThrow().getCustomMeta().isEmpty(), + "An explicit clear must not fall back to the earlier local focal point"); + verify(files, never()).retrieveMetaData(any()); + + doThrow(new DotDataException("S3 unavailable")).when(files) + .retrieveRawMetaData(argThat(key -> key.getStorage() == StorageType.S3)); + assertThrows(DotDataException.class, () -> nodeA.getMetadata(id)); + assertThrows(DotDataException.class, + () -> nodeA.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of("credit", "Lost")))); + assertThrows(IllegalArgumentException.class, () -> nodeA.getMetadata("temp_../other")); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, oldFlag); + } + } + + @Test + void disabledWritesRetainFilesystemPathAndProvider() throws Exception { + final String oldFlag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + final var files = mock(FileStorageAPI.class); + try (var locator = mockStatic(APILocator.class); var caches = mockStatic(CacheLocator.class); + var paths = mockStatic(ConfigUtils.class)) { + Config.setProperty(AssetStorageFeature.FLAG, false); + locator.when(APILocator::getFileStorageAPI).thenReturn(files); + caches.when(CacheLocator::getMetadataCache).thenReturn(mock(MetadataCache.class)); + paths.when(ConfigUtils::getAssetTempPath).thenReturn("/shared/nfs/tmp_upload"); + new FileMetadataAPIImpl().putCustomMetadataAttributes("temp_abc", Map.of("field", Map.of("credit", "Author"))); + verify(files).putCustomMetadataAttributes(argThat(request -> + request.getStorageKey().getStorage() == StorageType.FILE_SYSTEM + && request.getStorageKey().getPath().equals("/shared/nfs/tmp_upload/temp_abc/temp_abc.meta.tmp")), any()); + verify(files, never()).retrieveRawMetaData(any()); + verify(files, never()).setMetadata(any(), any()); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, oldFlag); + } + } +} diff --git a/dotCMS/src/test/java/com/dotmarketing/webdav/WebdavAssetStorageTest.java b/dotCMS/src/test/java/com/dotmarketing/webdav/WebdavAssetStorageTest.java new file mode 100644 index 000000000000..252068c3b5d7 --- /dev/null +++ b/dotCMS/src/test/java/com/dotmarketing/webdav/WebdavAssetStorageTest.java @@ -0,0 +1,184 @@ +package com.dotmarketing.webdav; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.dotcms.storage.AssetStorageFeature; +import com.dotcms.storage.binary.BinaryAssetStorageAPI; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.business.CacheLocator; +import com.dotmarketing.exception.DotRuntimeException; +import com.dotmarketing.portlets.contentlet.model.Contentlet; +import com.dotmarketing.portlets.fileassets.business.FileAsset; +import com.dotmarketing.portlets.fileassets.business.FileAssetAPI; +import com.dotmarketing.portlets.languagesmanager.business.LanguageAPI; +import com.dotmarketing.portlets.languagesmanager.model.Language; +import com.dotmarketing.util.Config; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class WebdavAssetStorageTest { + @Test + void uploadsWithTheSameNameHaveIndependentStagingFiles() throws Exception { + final String previous = Config.getStringProperty(AssetStorageFeature.FLAG, null); + try (var locator = mockStatic(APILocator.class)) { + final var files = mock(FileAssetAPI.class); + when(files.getRealAssetPathTmpBinary()).thenReturn(root.toString()); + locator.when(APILocator::getFileAssetAPI).thenReturn(files); + final var helper = mock(DotWebdavHelper.class, CALLS_REAL_METHODS); + final var field = mock(com.dotmarketing.portlets.structure.model.Field.class); + when(field.getFieldContentlet()).thenReturn("asset"); + Config.setProperty(AssetStorageFeature.FLAG, true); + final var first = helper.createFileInTemporalFolder(field, "same-user", "Mixed-Name.TXT"); + Files.writeString(first.toPath(), "first request"); + final var second = helper.createFileInTemporalFolder(field, "same-user", "Mixed-Name.TXT"); + Files.writeString(second.toPath(), "second request"); + assertNotEquals(first, second); + assertEquals(first.getName(), second.getName()); + assertEquals("first request", Files.readString(first.toPath())); + assertEquals("second request", Files.readString(second.toPath())); + assertThrows(IOException.class, () -> helper.createFileInTemporalFolder(field, "user", "../escape")); + + Config.setProperty(AssetStorageFeature.FLAG, false); + final var legacy = helper.createFileInTemporalFolder(field, "same-user", "Mixed-Name.TXT"); + Files.writeString(legacy.toPath(), "legacy staging"); + assertEquals(legacy, helper.createFileInTemporalFolder(field, "same-user", "Mixed-Name.TXT")); + assertFalse(legacy.exists(), "Disabled mode retains the existing staging behavior"); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, previous); + } + } + + @Test + void incompleteTemporaryUploadsDoNotReplaceCompleteFiles() throws Exception { + try (var config = mockStatic(com.dotmarketing.util.ConfigUtils.class)) { + config.when(com.dotmarketing.util.ConfigUtils::getAssetTempPath).thenReturn(root.toString()); + final var target = root.resolve("example.com/(Staging)/Mixed-Name.TXT"); + DotWebdavHelper.writeCompletedTempFile(target.toFile(), new ByteArrayInputStream("complete".getBytes())); + final var interrupted = new java.io.InputStream() { + private boolean first = true; + @Override public int read() throws IOException { + if (first) { first = false; return 'x'; } + throw new IOException("interrupted upload"); + } + }; + assertThrows(IOException.class, () -> DotWebdavHelper.writeCompletedTempFile(target.toFile(), interrupted)); + assertEquals("complete", Files.readString(target)); + final var absent = target.resolveSibling("Absent.TXT"); + assertThrows(IOException.class, () -> DotWebdavHelper.writeCompletedTempFile(absent.toFile(), interrupted)); + assertFalse(Files.exists(absent)); + try (var entries = Files.list(root)) { + assertFalse(entries.anyMatch(p -> p.getFileName().toString().startsWith(".webdav-write-"))); + } + assertThrows(IOException.class, () -> DotWebdavHelper.writeCompletedTempFile( + root.resolve("../outside.txt").toFile(), new ByteArrayInputStream(new byte[0]))); + final var link = root.resolve("link.txt"); + Files.createSymbolicLink(link, target); + assertThrows(IOException.class, () -> DotWebdavHelper.writeCompletedTempFile( + link.toFile(), new ByteArrayInputStream(new byte[0]))); + assertEquals("complete", Files.readString(target)); + } + } + + @TempDir Path root; + + @Test + void fileAssetSnapshotDoesNotInvokeItsMetadataGeneratingMap() throws Exception { + final String previous = Config.getStringProperty(AssetStorageFeature.FLAG, null); + final var asset = new FileAsset() { + @Override public Map getMap() { + throw new IllegalStateException("FileAsset map calculates metadata"); + } + }; + asset.setInode("abc123"); + try { + Config.setProperty(AssetStorageFeature.FLAG, true); + assertEquals("abc123", new Contentlet(asset).getInode()); + Config.setProperty(AssetStorageFeature.FLAG, false); + assertThrows(IllegalStateException.class, () -> new Contentlet(asset)); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, previous); + } + } + + @Test + void streamsAndHistoricalSizeChecksResolveUnderEvictionLease() throws Exception { + final String previous = Config.getStringProperty(AssetStorageFeature.FLAG, null); + try (var locator = mockStatic(APILocator.class); var caches = mockStatic(CacheLocator.class)) { + Config.setProperty(AssetStorageFeature.FLAG, true); + final var storage = mock(BinaryAssetStorageAPI.class); + final var held = new AtomicInteger(); + when(storage.acquireCacheLease()).thenAnswer(call -> { + held.incrementAndGet(); + return (BinaryAssetStorageAPI.CacheLease) held::decrementAndGet; + }); + locator.when(APILocator::getBinaryAssetStorageAPI).thenReturn(storage); + final var language = mock(LanguageAPI.class); + when(language.getDefaultLanguage()).thenReturn(new Language()); + locator.when(APILocator::getLanguageAPI).thenReturn(language); + locator.when(APILocator::getFileAssetAPI).thenReturn(mock(FileAssetAPI.class)); + final Path binary = Files.writeString(root.resolve("Hero-Mixed.GIF"), "original bytes"); + final FileAsset asset = mock(FileAsset.class); + when(asset.getBinary(FileAssetAPI.BINARY_FIELD)).thenAnswer(call -> { + assertTrue(held.get() > 0, "Resolve and open/stat must share the eviction lease"); + return binary.toFile(); + }); + doCallRealMethod().when(asset).getBinaryStream(FileAssetAPI.BINARY_FIELD); + try (var input = asset.getBinaryStream(FileAssetAPI.BINARY_FIELD)) { + assertEquals(0, held.get(), "The open descriptor can be consumed after releasing the lease"); + Files.delete(binary); + assertEquals("original bytes", new String(input.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + Files.writeString(binary, "restored bytes"); + final var resource = new FileResourceImpl(asset, "/webdav/working/1/example.com/Hero-Mixed.GIF"); + final var output = new ByteArrayOutputStream(); + resource.sendContent(output, null, Map.of(), null); + assertEquals("restored bytes", output.toString(java.nio.charset.StandardCharsets.UTF_8)); + assertEquals(Files.size(binary), resource.getContentLength()); + assertTrue(DotWebdavHelper.storedBinaryLength(asset) > 0); + Files.write(binary, new byte[0]); + assertEquals(0, DotWebdavHelper.storedBinaryLength(asset)); + Files.delete(binary); + assertThrows(IOException.class, () -> DotWebdavHelper.storedBinaryLength(asset)); + assertThrows(DotRuntimeException.class, resource::getContentLength); + assertEquals(0, held.get(), "Failures must also release the eviction lease"); + doReturn(null).when(asset).getBinary(FileAssetAPI.BINARY_FIELD); + assertEquals(-1, DotWebdavHelper.storedBinaryLength(asset), "Missing storage cannot justify deleting a version"); + assertThrows(IOException.class, () -> asset.getBinaryStream(FileAssetAPI.BINARY_FIELD)); + + final var stream = spy(new ByteArrayInputStream(new byte[] {1, 2, 3})); + doReturn(stream).when(asset).getBinaryStream(FileAssetAPI.BINARY_FIELD); + assertThrows(IOException.class, () -> resource.sendContent(new OutputStream() { + @Override public void write(int value) throws IOException { throw new IOException("disconnected"); } + }, null, Map.of(), null)); + verify(stream).close(); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, previous); + } + } + + @Test + void disabledBinaryStreamRetainsFilesystemBehaviorWithoutStorageInitialization() throws Exception { + final String previous = Config.getStringProperty(AssetStorageFeature.FLAG, null); + try (var locator = mockStatic(APILocator.class)) { + Config.setProperty(AssetStorageFeature.FLAG, false); + final var content = new Contentlet(); + final Path source = Files.writeString(root.resolve("NFS-Mixed.TXT"), "shared filesystem"); + content.getMap().put(FileAssetAPI.BINARY_FIELD, source.toFile()); + try (var input = content.getBinaryStream(FileAssetAPI.BINARY_FIELD)) { + assertEquals("shared filesystem", new String(input.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)); + } + locator.verifyNoInteractions(); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, previous); + } + } +} diff --git a/dotcms-integration/src/test/java/com/dotcms/MainSuite2a.java b/dotcms-integration/src/test/java/com/dotcms/MainSuite2a.java index 665a16cb2dfc..6a82f2c2412b 100644 --- a/dotcms-integration/src/test/java/com/dotcms/MainSuite2a.java +++ b/dotcms-integration/src/test/java/com/dotcms/MainSuite2a.java @@ -19,6 +19,7 @@ import com.dotcms.junit.MainBaseSuite; import com.dotcms.rest.api.v1.maintenance.MaintenanceResourceIntegrationTest; import com.dotmarketing.portlets.containers.business.ContainerStructureFinderStrategyResolverTest; +import com.dotmarketing.webdav.DotWebdavHelperTest; import org.junit.runner.RunWith; import org.junit.runners.Suite.SuiteClasses; @@ -102,7 +103,8 @@ com.dotcms.rest.AuditPublishingResourceTest.class, MaintenanceResourceIntegrationTest.class, FolderHandlerTest.class, - ContentAnalyticsPersistenceModeIT.class + ContentAnalyticsPersistenceModeIT.class, + DotWebdavHelperTest.class }) public class MainSuite2a { diff --git a/dotcms-integration/src/test/java/com/dotmarketing/image/focalpoint/FocalPointAPITest.java b/dotcms-integration/src/test/java/com/dotmarketing/image/focalpoint/FocalPointAPITest.java index dc934b25b1b5..d01de8b1e68c 100644 --- a/dotcms-integration/src/test/java/com/dotmarketing/image/focalpoint/FocalPointAPITest.java +++ b/dotcms-integration/src/test/java/com/dotmarketing/image/focalpoint/FocalPointAPITest.java @@ -163,10 +163,13 @@ public void Test_Write_Focal_Point_From_Temp_Asset_Id() throws Exception { assertEquals(focalPoint2.get(),focalPointSrc); final HttpServletRequest request = mockHttpServletRequest(); - final DotTempFile dotTempFile = APILocator.getTempFileAPI().createEmptyTempFile("temp", request); + DotTempFile dotTempFile = APILocator.getTempFileAPI().createEmptyTempFile("temp", request); assertTrue(dotTempFile.file.createNewFile()); assertTrue(dotTempFile.file.setLastModified(System.currentTimeMillis())); assertTrue(dotTempFile.file.exists()); + // Finish the upload as the upload API and image editor do; with S3 asset storage enabled an + // unfinished upload is not yet a usable temp resource. + dotTempFile = APILocator.getTempFileAPI().completeTempFile(dotTempFile); focalPointAPI.writeFocalPoint(dotTempFile.id,"fileAsset", focalPointSrc); assertTrue(focalPointAPI.readFocalPoint(dotTempFile.id, "fileAsset").isPresent()); @@ -203,9 +206,11 @@ public void Test_Persist_Focal_Point_On_Temp_From_Save_Params() throws Exception // 2. A temp file is generated for the edited image. final HttpServletRequest request = mockHttpServletRequest(); - final DotTempFile dotTempFile = APILocator.getTempFileAPI().createEmptyTempFile("temp", request); + DotTempFile dotTempFile = APILocator.getTempFileAPI().createEmptyTempFile("temp", request); assertTrue(dotTempFile.file.createNewFile()); assertTrue(dotTempFile.file.setLastModified(System.currentTimeMillis())); + // The servlet completes the temp before writing the focal point onto it. + dotTempFile = APILocator.getTempFileAPI().completeTempFile(dotTempFile); // 3. The servlet writes the focal point straight onto that temp (independent of the filter). focalPointAPI.writeFocalPoint(dotTempFile.id, fieldVar, parsed.get()); diff --git a/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java b/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java index f8bd212ce34c..03f99f3342a3 100644 --- a/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java +++ b/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java @@ -36,6 +36,235 @@ public static void prepare() throws Exception { private final DotWebdavHelper helper = new DotWebdavHelper(); + @Test + public void completedTemporaryUploadsAreDiscoverableAndSurviveInterruptedReplacement() throws Exception { + org.junit.Assume.assumeTrue(com.dotcms.storage.AssetStorageFeature.isEnabled()); + final Host host = APILocator.getHostAPI().findDefaultHost(APILocator.systemUser(), false); + final Folder folder = new FolderDataGen().site(host).name("Mixed-Dav-" + java.util.UUID.randomUUID()).nextPersisted(); + final String base = "/webdav/working/1/" + host.getHostname() + "/" + folder.getName() + "/"; + final var temporaryDirectory = helper.loadTempFile(base).toPath(); + try (var requests = Mockito.mockStatic(HttpManager.class)) { + final var request = Mockito.mock(Request.class); + final var auth = Mockito.mock(com.bradmcevoy.http.Auth.class); + when(auth.getTag()).thenReturn(APILocator.systemUser()); + when(request.getAuthorization()).thenReturn(auth); + requests.when(HttpManager::request).thenReturn(request); + final var normalFolder = (FolderResourceImpl) helper.getResourceFromURL("http://localhost:8080" + base); + Assert.assertNotNull(normalFolder); + final var complete = (TempFileResourceImpl) normalFolder.createNew(".Mixed-Name.TXT", + new ByteArrayInputStream("complete".getBytes()), 8L, "text/plain"); + Assert.assertEquals(".Mixed-Name.TXT", complete.getName()); + java.nio.file.Files.delete(complete.getFile().toPath()); + java.nio.file.Files.delete(helper.loadTempFile(base + ".Mixed-Name.TXT").toPath()); + Assert.assertEquals(Long.valueOf(8), complete.getContentLength()); + Assert.assertEquals("complete", java.nio.file.Files.readString(complete.getFile().toPath())); + Assert.assertNotNull(helper.getResourceFromURL("http://localhost:8080" + base + ".Mixed-Name.TXT")); + Assert.assertNotNull(helper.getResourceFromURL("http://localhost:8080" + base.toLowerCase(java.util.Locale.ROOT) + ".Mixed-Name.TXT")); + Assert.assertTrue(helper.getChildrenOfFolder(folder, APILocator.systemUser(), false, 1).stream() + .anyMatch(resource -> resource.getName().equals(".Mixed-Name.TXT"))); + Assert.assertNotNull(normalFolder.child(".Mixed-Name.TXT")); + Assert.assertNull(normalFolder.child(".mixed-name.txt")); + final var routedDirectory = (TempFolderResourceImpl) normalFolder.createCollection("(Mixed-Staging)"); + Assert.assertEquals("(Mixed-Staging)", routedDirectory.getFolder().getName()); + final var routedChild = (TempFileResourceImpl) routedDirectory.createNew("Nested-Mixed.TXT", + new ByteArrayInputStream("routed".getBytes()), 6L, "text/plain"); + java.nio.file.Files.delete(routedChild.getFile().toPath()); + FileUtil.deltree(routedDirectory.getFolder(), true); + final var resolvedChild = (TempFileResourceImpl) helper.getResourceFromURL( + "http://localhost:8080" + base.toLowerCase(java.util.Locale.ROOT) + "(Mixed-Staging)/Nested-Mixed.TXT"); + Assert.assertNotNull(resolvedChild); + Assert.assertEquals("routed", java.nio.file.Files.readString(resolvedChild.getFile().toPath())); + Assert.assertNull(helper.getResourceFromURL("http://localhost:8080" + base + "(Mixed-Staging)/nested-mixed.txt")); + Assert.assertTrue(helper.getChildrenOfFolder(folder, APILocator.systemUser(), false, 1).stream() + .anyMatch(resource -> resource.getName().equals("(Mixed-Staging)"))); + routedDirectory.delete(); + Assert.assertThrows(com.dotmarketing.exception.DotRuntimeException.class, () -> + normalFolder.createNew(".Mixed-Name.TXT", failingUpload(), 10L, "text/plain")); + Assert.assertEquals("complete", java.nio.file.Files.readString(complete.getFile().toPath())); + Assert.assertThrows(com.dotmarketing.exception.DotRuntimeException.class, () -> + normalFolder.createNew(".Absent.TXT", failingUpload(), 10L, "text/plain")); + Assert.assertFalse(helper.loadTempFile(base + ".Absent.TXT").exists()); + + final var staging = (TempFolderResourceImpl) normalFolder.createCollection("(Staging)"); + final var nested = (TempFileResourceImpl) staging.createNew("Nested-Mixed.TXT", + new ByteArrayInputStream("nested".getBytes()), 6L, "text/plain"); + Assert.assertThrows(IOException.class, () -> + staging.createNew("Nested-Mixed.TXT", failingUpload(), 10L, "text/plain")); + final var output = new java.io.ByteArrayOutputStream(); + nested.sendContent(output, null, java.util.Map.of(), null); + Assert.assertEquals("nested", output.toString(java.nio.charset.StandardCharsets.UTF_8)); + java.nio.file.Files.delete(nested.getFile().toPath()); + FileUtil.deltree(temporaryDirectory.toFile(), true); + Assert.assertNotNull(staging.child("Nested-Mixed.TXT")); + Assert.assertNull(staging.child("nested-mixed.txt")); + final var empty = (TempFolderResourceImpl) staging.createCollection("Empty"); + Assert.assertTrue(empty.getChildren().isEmpty()); + nested.copyTo(empty, "Copied.TXT"); + final var copied = (TempFileResourceImpl) empty.child("Copied.TXT"); + Assert.assertEquals("nested", java.nio.file.Files.readString(copied.getFile().toPath())); + copied.moveTo(empty, "Moved.TXT"); + Assert.assertNull(empty.child("Copied.TXT")); + final var moved = (TempFileResourceImpl) empty.child("Moved.TXT"); + moved.moveTo(empty, "Moved.TXT"); + Assert.assertNotNull(empty.child("Moved.TXT")); + empty.copyTo(staging, "Folder-Copy"); + Assert.assertNotNull(((TempFolderResourceImpl) staging.child("Folder-Copy")).child("Moved.TXT")); + empty.moveTo(staging, "Folder-Moved"); + Assert.assertNull(staging.child("Empty")); + final var folderMoved = (TempFolderResourceImpl) staging.child("Folder-Moved"); + folderMoved.moveTo(staging, "Folder-Moved"); + Assert.assertNotNull(staging.child("Folder-Moved")); + folderMoved.delete(); + Assert.assertNull(staging.child("Folder-Moved")); + staging.delete(); + complete.delete(); + Assert.assertNull(helper.getResourceFromURL("http://localhost:8080" + base + ".Mixed-Name.TXT")); + } finally { + com.dotcms.storage.WebdavTemporaryStorage.getInstance().delete(temporaryDirectory.toFile()); + FileUtil.deltree(temporaryDirectory.toFile(), true); + FolderDataGen.remove(folder); + } + } + + private static InputStream failingUpload() { + return new InputStream() { + private boolean first = true; + @Override public int read() throws IOException { + if (first) { first = false; return 'x'; } + throw new IOException("interrupted WebDAV upload"); + } + }; + } + + @Test + public void siteAndLanguageRootsDiscoverRemoteTemporaryChildren() throws Exception { + org.junit.Assume.assumeTrue(com.dotcms.storage.AssetStorageFeature.isEnabled()); + final Host host = APILocator.getHostAPI().findDefaultHost(APILocator.systemUser(), false); + final String name = "(Mixed-Staging-" + java.util.UUID.randomUUID() + ")"; + final String sitePath = "/webdav/working/1/" + host.getHostname(); + final String languagePath = "/webdav/live/1/system/languages"; + try (var requests = Mockito.mockStatic(HttpManager.class)) { + final var request = Mockito.mock(Request.class); + final var auth = Mockito.mock(com.bradmcevoy.http.Auth.class); + when(auth.getTag()).thenReturn(APILocator.systemUser()); + when(request.getAuthorization()).thenReturn(auth); + requests.when(HttpManager::request).thenReturn(request); + final var site = new HostResourceImpl(sitePath); + final var languages = new LanguageFolderResourceImpl(""); + final var siteTemp = (TempFolderResourceImpl) site.createCollection(name); + final var languageTemp = (TempFolderResourceImpl) languages.createCollection(name); + try { + for (var temporary : java.util.List.of(siteTemp, languageTemp)) { + final var child = (TempFileResourceImpl) temporary.createNew("Mixed.TXT", + new ByteArrayInputStream("remote bytes".getBytes()), 12L, "text/plain"); + java.nio.file.Files.delete(child.getFile().toPath()); + FileUtil.deltree(temporary.getFolder(), true); + } + Assert.assertTrue(site.getChildren().stream().anyMatch(resource -> resource.getName().equals(name))); + Assert.assertNotNull(site.child(name)); + Assert.assertNull(site.child(name.toLowerCase(java.util.Locale.ROOT))); + Assert.assertTrue(languages.getChildren().stream().anyMatch(resource -> resource.getName().equals(name))); + Assert.assertNotNull(languages.child(name)); + Assert.assertNull(languages.child(name.toLowerCase(java.util.Locale.ROOT))); + for (String parent : java.util.List.of(sitePath, languagePath)) { + final var child = (TempFileResourceImpl) helper.getResourceFromURL( + "http://localhost:8080" + parent + "/" + name + "/Mixed.TXT"); + Assert.assertNotNull(child); + Assert.assertEquals("remote bytes", java.nio.file.Files.readString(child.getFile().toPath())); + } + siteTemp.delete(); + languageTemp.delete(); + Assert.assertNull(site.child(name)); + Assert.assertNull(languages.child(name)); + } finally { + siteTemp.delete(); + languageTemp.delete(); + } + } + } + + /** + * Evicts a local cache file, retrying for a few seconds. Eviction deliberately declines while any + * cache lease is held, for example by a background reindex reading the file just checked in, so a + * single attempt can fail for reasons unrelated to the behavior under test. + * + * @param binaries the binary asset API + * @param file the local cache file to evict + * @return {@code true} once the file is evicted, {@code false} if it never was within the timeout + */ + private static boolean evictEventually(final com.dotcms.storage.binary.BinaryAssetStorageAPI binaries, + final java.io.File file) throws Exception { + final long deadline = System.currentTimeMillis() + 10_000; + while (!binaries.evictLocalFile(file)) { + if (System.currentTimeMillis() > deadline) { + return false; + } + Thread.sleep(100); + } + return true; + } + + @Test + public void uploadColdReadCopyAndOverwriteRetainHistoricalBytes() throws Exception { + final boolean enabled = com.dotcms.storage.AssetStorageFeature.isEnabled(); + Assert.assertEquals(Boolean.getBoolean("s3.cms.enabled"), enabled); + final Host host = APILocator.getHostAPI().findDefaultHost(APILocator.systemUser(), false); + final Folder folder = new FolderDataGen().site(host).nextPersisted(); + final var user = APILocator.systemUser(); + final var binaries = APILocator.getBinaryAssetStorageAPI(); + final String base = "/webdav/working/1/" + host.getHostname() + "/" + folder.getName() + "/"; + final String source = base + "Mixed-Case.TXT"; + final byte[] original = "original WebDAV asset".getBytes(java.nio.charset.StandardCharsets.UTF_8); + try (var requests = Mockito.mockStatic(HttpManager.class)) { + final var request = Mockito.mock(Request.class); + when(request.getUserAgentHeader()).thenReturn("WebDAV S3 acceptance"); + requests.when(HttpManager::request).thenReturn(request); + try (var upload = new ByteArrayInputStream(original)) { + helper.setResourceContent(source, upload, "text/plain", null, new java.util.Date(), user, false); + } + final var first = helper.loadFile(source, user); + Assert.assertNotNull(first); + final var firstContent = (com.dotmarketing.portlets.contentlet.model.Contentlet) first; + final String firstInode = firstContent.getInode(); + final var firstFile = firstContent.getBinary(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD); + Assert.assertEquals("Mixed-Case.TXT", firstFile.getName()); + if (enabled) Assert.assertTrue(evictEventually(binaries, firstFile)); + final var output = new java.io.ByteArrayOutputStream(); + new FileResourceImpl(first, source).sendContent(output, null, java.util.Map.of(), null); + Assert.assertArrayEquals(original, output.toByteArray()); + + if (enabled) Assert.assertTrue(evictEventually(binaries, firstFile)); + helper.copyResource(source, base + "Copy-Mixed.TXT", user, enabled); + final var copy = (com.dotmarketing.portlets.contentlet.model.Contentlet) helper.loadFile(base + "Copy-Mixed.TXT", user); + Assert.assertNotNull(copy); + if (enabled) Assert.assertTrue(copy.isLive()); + final var copyFile = copy.getBinary(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD); + if (enabled) Assert.assertTrue(evictEventually(binaries, copyFile)); + try (var input = copy.getBinaryStream(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD)) { + Assert.assertArrayEquals(original, input.readAllBytes()); + } + + if (enabled) Assert.assertTrue(evictEventually(binaries, firstContent.getBinary(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD))); + try (var upload = new ByteArrayInputStream("replacement".getBytes(java.nio.charset.StandardCharsets.UTF_8))) { + helper.setResourceContent(source, upload, "text/plain", null, new java.util.Date(), user, false); + } + final var historical = APILocator.getContentletAPI().find(firstInode, user, false); + Assert.assertNotNull(historical); + try (var input = historical.getBinaryStream(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD)) { + Assert.assertArrayEquals(original, input.readAllBytes()); + } + if (enabled) { + try (var upload = new ByteArrayInputStream(original)) { + org.junit.Assert.assertThrows(IOException.class, () -> helper.setResourceContent( + "/webdav/working/1/" + host.getHostname() + "/missing-" + java.util.UUID.randomUUID() + "/file.txt", + upload, "text/plain", null, new java.util.Date(), user, false)); + } + } + } finally { + FolderDataGen.remove(folder); + } + } + @Test public void Test_Get_Folder_Resource_Then_Get_File_Resource() throws IOException, DotDataException, DotSecurityException { final SiteDataGen siteDataGen = new SiteDataGen(); From 066326b4cc67bcc046a8570de9685d1785a7caa0 Mon Sep 17 00:00:00 2001 From: Scott Wicken <1562170+swicken@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:49:15 -0400 Subject: [PATCH 2/4] docs(storage): document that WebDAV temporary-file tombstones are not reclaimed --- docs/testing/BINARY_S3_STORAGE.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/testing/BINARY_S3_STORAGE.md b/docs/testing/BINARY_S3_STORAGE.md index c326c543e530..127b628d8b02 100644 --- a/docs/testing/BINARY_S3_STORAGE.md +++ b/docs/testing/BINARY_S3_STORAGE.md @@ -354,9 +354,14 @@ through a later replacement. Mixed-case temporary names are kept, while normal C case-insensitive handling. Cleanup uses S3 server timestamps, protects completed and pending payload references, and -conditionally replaces expired path records with tombstones. Normal WebDAV uploads still go through -content check-in, and their reads, copies and overwrites hold the cache lease through the stream. -With the flag off, WebDAV and temporary uploads use the local filesystem as before. +conditionally replaces expired path records with tombstones. Tombstones are never deleted, because +some S3-compatible stores (MinIO among them) ignore the condition on a conditional delete, and an +unconditional delete could remove a newer upload that reused the path. The group therefore keeps +one small tombstone record per deleted WebDAV temporary path, and each cleanup run reads them +again, until safe reclamation is added. Normal WebDAV uploads still go through content check-in, +and their reads, copies and overwrites open the file under the cache lease through +`Contentlet.getBinaryStream`. With the flag off, WebDAV and temporary uploads use the local +filesystem as before. ## Local cache eviction From 668fa15b87709c09db50a96eab263d308791d384 Mon Sep 17 00:00:00 2001 From: Scott Wicken <1562170+swicken@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:08:41 -0400 Subject: [PATCH 3/4] refactor(storage): take the temporary upload and WebDAV remote store from the storage provider --- .../java/com/dotcms/storage/FileMetadataAPIImpl.java | 2 +- .../java/com/dotcms/storage/TemporaryAssetStorage.java | 4 ++-- .../com/dotcms/storage/WebdavTemporaryStorage.java | 10 +++++----- .../java/com/dotcms/storage/BinaryS3StorageTest.java | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java b/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java index d69a9e74f83d..260bafbfce02 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java @@ -1071,7 +1071,7 @@ private StorageKey temporaryMetadataKey(final String id, final boolean legacy) { } return new StorageKey.Builder().group(Config.getStringProperty(METADATA_GROUP_NAME, DOT_METADATA)) .path(legacy ? tempResourcePath(id) : TemporaryAssetStorage.metadataPath(id)) - .storage(legacy ? StorageType.FILE_SYSTEM : StorageType.S3).build(); + .storage(legacy ? StorageType.FILE_SYSTEM : StoragePersistenceProvider.remoteStorageType()).build(); } private Map temporaryMetadata(final String id) throws DotDataException { diff --git a/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java b/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java index 57271addab6b..ffc49787152e 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java +++ b/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java @@ -90,7 +90,7 @@ public boolean isManagedLocally(final String id) { private synchronized StoragePersistenceAPI remote() throws DotDataException { if (remote == null) { - remote = AmazonS3StoragePersistenceAPIImpl.withPlainPaths(); + remote = StoragePersistenceProvider.remoteObjectStorage(); } return remote; } @@ -201,7 +201,7 @@ public Optional retrieve(final Receipt record, final List accessin private void removeMetadataAndRenditions(final String id) throws DotDataException { APILocator.getFileStorageAPI().removeMetaData(new FetchMetadataParams.Builder().cache(false) - .storageKey(new StorageKey.Builder().storage(StorageType.S3) + .storageKey(new StorageKey.Builder().storage(StoragePersistenceProvider.remoteStorageType()) .group(Config.getStringProperty(StoragePersistenceProvider.METADATA_GROUP_NAME, FileMetadataAPI.DOT_METADATA)) .path(metadataPath(id)).build()).build()); APILocator.getBinaryAssetStorageAPI().deleteGeneratedFiles(id); diff --git a/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java b/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java index e1075b9680ac..2ade41535464 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java +++ b/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java @@ -22,7 +22,7 @@ public final class WebdavTemporaryStorage { public static final String GROUP = "webdav-temporary"; private static final JsonReaderDelegate READER = new JsonReaderDelegate<>(Map.class); private final Path configuredRoot; - private AmazonS3StoragePersistenceAPIImpl remote; + private StoragePersistenceAPI remote; private boolean groupReady; private static class Holder { @@ -31,7 +31,7 @@ private static class Holder { public static WebdavTemporaryStorage getInstance() { return Holder.INSTANCE; } - public WebdavTemporaryStorage(AmazonS3StoragePersistenceAPIImpl remote, Path root) { + public WebdavTemporaryStorage(StoragePersistenceAPI remote, Path root) { this.remote = remote; this.configuredRoot = root; } @@ -66,9 +66,9 @@ public File file(String key) throws IOException { return file; } - private synchronized AmazonS3StoragePersistenceAPIImpl remote() throws DotDataException { + private synchronized StoragePersistenceAPI remote() throws DotDataException { if (!AssetStorageFeature.isEnabled()) throw new DotDataException("S3 asset storage is disabled"); - if (remote == null) remote = AmazonS3StoragePersistenceAPIImpl.withPlainPaths(); + if (remote == null) remote = StoragePersistenceProvider.remoteObjectStorage(); if (!groupReady) { remote.createGroup(GROUP); groupReady = true; @@ -99,7 +99,7 @@ private String recordPath(String object) throws IOException { return path; } - private record State(AmazonS3StoragePersistenceAPIImpl.ObjectSnapshot snapshot, Entry entry, String pending) { } + private record State(ObjectSnapshot snapshot, Entry entry, String pending) { } private State state(String key) throws DotDataException, IOException { file(key); diff --git a/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java b/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java index 1f037b7870b7..9db9ba7209ac 100644 --- a/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java +++ b/dotCMS/src/test/java/com/dotcms/storage/BinaryS3StorageTest.java @@ -402,9 +402,9 @@ private void assertWebdavTombstones() throws Exception { private static void pinWebdavCleanupClock(AmazonS3StoragePersistenceAPIImpl remote, java.util.concurrent.atomic.AtomicLong time) throws Exception { doAnswer(call -> { - final var actual = (AmazonS3StoragePersistenceAPIImpl.ObjectSnapshot) call.callRealMethod(); + final var actual = (ObjectSnapshot) call.callRealMethod(); if (actual != null && actual.path().startsWith("maintenance/") && time.get() != 0) { - return new AmazonS3StoragePersistenceAPIImpl.ObjectSnapshot(actual.path(), actual.value(), actual.version(), time.get()); + return new ObjectSnapshot(actual.path(), actual.value(), actual.version(), time.get()); } return actual; }).when(remote).readObjectSnapshot(anyString(), anyString(), any()); From e7d3e70fcb2db01900e72296c442bc9fe1abcaa1 Mon Sep 17 00:00:00 2001 From: Scott Wicken <1562170+swicken@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:55:14 -0400 Subject: [PATCH 4/4] fix(storage): keep WebDAV copies unpublished and make temp-upload and WebDAV listing cleanup robust These fixes apply only with FEATURE_FLAG_S3_ASSET_STORAGE on; flag-off behavior is unchanged. WebDAV COPY of a CMS file no longer passes the caller's auto-publish flag on the S3 path. The copy is saved as an unpublished working version, as on main, so it needs only edit permission. The stream closing that the S3 path added is kept. Focal points that the image filter stores under a temp_ id stay in the local temporary directory, where BinaryCleanupJob removes them as before. Those ids never get an upload receipt, so S3 records written for them were never cleaned up. The unreachable flag check in the legacy metadata loop is removed. A WebDAV folder listing builds each temporary file resource from the entry the listing already read, instead of looking it up again, so a temporary file deleted during the listing no longer fails it. If S3 cannot be read, the failure is logged and only the temporary children are left out, so the CMS files and folders still list. WebdavTemporaryStorage.children reads only the records that decide each direct child: the child's own record, and descendants only for a folder with no live record of its own. Records below a live folder are no longer fetched on every listing. Expired temporary uploads have their S3 objects removed at the TTL, but their local copies are left to the existing CLEANUP_TMP_FILES_OLDER_THAN_HOURS age rule, so a check-in that resolved the file just before expiry keeps its source. Temporary-upload cleanup handles each upload separately. A failure is logged, that upload keeps its receipt and payload for the next run, the rest are still cleaned, and one combined failure is thrown so the job still reports it. Adds WebdavTemporaryStorageTest (in-memory remote) for the listing fixes, and unit tests for the focal-point routing and the per-upload cleanup. DotWebdavHelperTest now expects an unpublished copy and no longer reads the s3.cms.enabled system property. The storage doc describes the new behavior, the remaining listing cost next to the tombstone note, and the accepted metadata race. --- docs/testing/BINARY_S3_STORAGE.md | 32 +++- .../dotcms/storage/FileMetadataAPIImpl.java | 63 +++++-- .../dotcms/storage/TemporaryAssetStorage.java | 60 +++++-- .../storage/WebdavTemporaryStorage.java | 51 ++++-- .../dotmarketing/webdav/DotWebdavHelper.java | 38 ++++- .../dotmarketing/webdav/HostResourceImpl.java | 28 ++-- .../webdav/LanguageFolderResourceImpl.java | 14 +- .../webdav/ResourceFactoryImpl.java | 13 +- .../webdav/TempFileResourceImpl.java | 46 +++++- .../webdav/TempFolderResourceImpl.java | 8 +- .../storage/TemporaryAssetStorageTest.java | 50 +++++- .../storage/TemporaryMetadataStorageTest.java | 35 ++++ .../webdav/WebdavTemporaryStorageTest.java | 154 ++++++++++++++++++ .../webdav/DotWebdavHelperTest.java | 5 +- 14 files changed, 521 insertions(+), 76 deletions(-) create mode 100644 dotCMS/src/test/java/com/dotmarketing/webdav/WebdavTemporaryStorageTest.java diff --git a/docs/testing/BINARY_S3_STORAGE.md b/docs/testing/BINARY_S3_STORAGE.md index 127b628d8b02..127241a53077 100644 --- a/docs/testing/BINARY_S3_STORAGE.md +++ b/docs/testing/BINARY_S3_STORAGE.md @@ -339,9 +339,22 @@ interrupted upload keeps any existing complete file and never exposes a partial uploads without a receipt cannot fall back to legacy local access. Custom metadata for a temporary upload, such as a focal point set before check-in, is stored with -the upload in S3 and read directly from S3, so an edit made on another node is visible. The -scheduled `BinaryCleanupJob` also removes expired S3 temporary uploads, and a failure is logged and -retried on the next run. +the upload in S3 and read directly from S3, so an edit made on another node is visible. The image +filter's focal point for existing content uses an id made of `temp_` and the content inode. That id +never gets an upload receipt, so its metadata stays in the local temporary directory, as with the +flag off, where `BinaryCleanupJob` removes it after `CLEANUP_TMP_FILES_OLDER_THAN_HOURS`. It is +therefore visible only on the node that wrote it, which the sticky sessions a cluster already +needs make sufficient. Metadata edits are a read, merge and unconditional write, so two nodes +setting different attributes on the same upload at the same moment can lose one of the two +writes. This is accepted because temporary metadata lives only until check-in. + +The scheduled `BinaryCleanupJob` removes the S3 objects of an expired upload (its receipt, payload, +metadata and renditions) once `TEMP_RESOURCE_MAX_AGE_SECONDS` has passed. Local copies are left to +the existing `CLEANUP_TMP_FILES_OLDER_THAN_HOURS` age rule, as with the flag off, so a check-in that +resolved the file just before it expired keeps its source; the local upload marker stops an expired +copy from being served. Each upload is cleaned on its own: a failure is logged, that upload keeps +its receipt and payload, the remaining uploads are still cleaned, and the job reports one combined +failure and retries on the next run. ## WebDAV temporary files @@ -363,6 +376,17 @@ and their reads, copies and overwrites open the file under the cache lease throu `Contentlet.getBinaryStream`. With the flag off, WebDAV and temporary uploads use the local filesystem as before. +A WebDAV COPY of a CMS file creates an unpublished working version on every mount, with either flag +setting, so it needs only edit permission. A folder listing (PROPFIND) reads only the records that +decide each direct temporary child: the child's own record, and only for a folder without a live +record of its own, its descendants until one live descendant is found. Each file resource is built +from the record the listing read, so a temporary file that a client deletes during the listing is +skipped instead of failing it. If S3 cannot be read, the failure is logged and only the temporary +children are left out, so the CMS files and folders still list. The remaining cost is one S3 read +per direct child record on every listing, and that includes the tombstone of every deleted direct +child, so it grows with the number of temporary names ever written directly in that folder until +tombstones can be reclaimed. + ## Local cache eviction With the flag on, the local asset directory is a cache that `BinaryCacheEvictionJob` can trim. @@ -461,7 +485,7 @@ docker run -d --rm --name binary-cleanup-postgres-test \ -e POSTGRES_DB=binary_storage_test postgres:16-alpine ./mvnw test -pl :dotcms-core -Dmaven.build.cache.enabled=false \ - -Dtest=AssetStorageFeatureTest,AssetStorageFeatureLatchTest,S3StorageConfigurationTest,NoWebIdentityCredentialsProviderChainTest,BinaryS3StorageTest,BinaryAssetReferenceTest,BinaryCacheEvictionJobTest,BinaryFileSystemStorageTest,BinaryAssetStorageAPIImplTest,MetadataLocalCacheTest,BinaryAssetCleanupTransactionTest,BinaryAssetCleanupProcessorTest,ContentletBackupStorageGateTest,BinaryFieldCleanupProcessorTest,AssetJobEventSerializationTest,BinaryAssetBackfillCheckpointTest,ImportStarterWorkflowCleanupTest,BinaryAssetBackfillProcessorTest,BinaryAssetBackfillTest,ExportStarterFailureTest,BundleArchiveStorageTest,FileAssetBundlerTest,TemporaryAssetStorageTest,WebdavAssetStorageTest,TemporaryMetadataStorageTest \ + -Dtest=AssetStorageFeatureTest,AssetStorageFeatureLatchTest,S3StorageConfigurationTest,NoWebIdentityCredentialsProviderChainTest,BinaryS3StorageTest,BinaryAssetReferenceTest,BinaryCacheEvictionJobTest,BinaryFileSystemStorageTest,BinaryAssetStorageAPIImplTest,MetadataLocalCacheTest,BinaryAssetCleanupTransactionTest,BinaryAssetCleanupProcessorTest,ContentletBackupStorageGateTest,BinaryFieldCleanupProcessorTest,AssetJobEventSerializationTest,BinaryAssetBackfillCheckpointTest,ImportStarterWorkflowCleanupTest,BinaryAssetBackfillProcessorTest,BinaryAssetBackfillTest,ExportStarterFailureTest,BundleArchiveStorageTest,FileAssetBundlerTest,TemporaryAssetStorageTest,WebdavAssetStorageTest,TemporaryMetadataStorageTest,WebdavTemporaryStorageTest \ -Ds3.test.endpoint=http://127.0.0.1:19002 \ -Ds3.test.jdbc=jdbc:postgresql://127.0.0.1:19003/binary_storage_test diff --git a/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java b/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java index 260bafbfce02..4b67040b4dc7 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java +++ b/dotCMS/src/main/java/com/dotcms/storage/FileMetadataAPIImpl.java @@ -24,12 +24,14 @@ import com.dotmarketing.business.CacheLocator; import com.dotmarketing.business.FactoryLocator; import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.image.focalpoint.FocalPointAPIImpl; import com.dotmarketing.portlets.contentlet.business.MetadataCache; import com.dotmarketing.portlets.contentlet.model.Contentlet; import com.dotmarketing.util.Config; import com.dotmarketing.util.ConfigUtils; import com.dotmarketing.util.Logger; import com.dotmarketing.util.UtilMethods; +import com.dotmarketing.util.UUIDUtil; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.liferay.util.StringPool; @@ -1074,23 +1076,57 @@ private StorageKey temporaryMetadataKey(final String id, final boolean legacy) { .storage(legacy ? StorageType.FILE_SYSTEM : StoragePersistenceProvider.remoteStorageType()).build(); } + /** + * Tells whether metadata for a temporary id is kept in S3 with the upload it describes. + * + *

An id made of {@code temp_} and a content inode does not name an upload. + * {@code FocalPointImageFilter} uses it to hold a focal point for existing content until the + * image editor saves. It never gets an upload receipt, so the S3 temporary-upload cleanup would + * never find it. Its metadata therefore stays in the local temporary directory, where + * {@code BinaryCleanupJob} ages it out, exactly as with the flag off. + * + * @param id the temporary resource id + * @return {@code false} for an inode-based id, {@code true} otherwise + */ + private static boolean isSharedTemporaryId(final String id) { + return id == null || !(id.startsWith(FocalPointAPIImpl.TMP) + && UUIDUtil.isUUID(id.substring(FocalPointAPIImpl.TMP.length()))); + } + + /** + * Reads the metadata of a temporary resource with the flag on. + * + *

Upload metadata is mutable, so it is read from S3 directly and another node's edits are + * visible. Only a genuine absence falls back to the local metadata file, never a storage + * failure. Inode-based ids (see {@link #isSharedTemporaryId(String)}) are read from the local + * file only. + * + * @param id the temporary resource id + * @return the stored metadata, or {@code null} if there is none + * @throws DotDataException if S3 cannot be read + */ private Map temporaryMetadata(final String id) throws DotDataException { - // Temporary metadata is mutable: read S3 directly so another node's edits are visible. - // Only a genuine absence falls back to pre-feature local metadata, never a storage failure. - final Map shared = fileStorageAPI.retrieveRawMetaData(temporaryMetadataKey(id, false)); + final Map shared = isSharedTemporaryId(id) + ? fileStorageAPI.retrieveRawMetaData(temporaryMetadataKey(id, false)) : null; return shared != null ? shared : fileStorageAPI.retrieveRawMetaData(temporaryMetadataKey(id, true)); } /** * {@inheritDoc} - * @param tempResourceId - * @param customAttributesByField - * @throws DotDataException + * + *

With S3 asset storage on, metadata for an upload is merged and written to S3, so every node + * sees it and the temporary-upload cleanup removes it with the upload. Inode-based ids (see + * {@link #isSharedTemporaryId(String)}) and the flag-off mode write the local metadata file. + * + * @param tempResourceId the temporary resource id + * @param customAttributesByField custom attributes to set, by field; an empty map for a field + * clears its custom attributes + * @throws DotDataException if the S3 metadata cannot be read or written */ public void putCustomMetadataAttributes(final String tempResourceId, final Map> customAttributesByField) throws DotDataException { - if (AssetStorageFeature.isEnabled()) { + if (AssetStorageFeature.isEnabled() && isSharedTemporaryId(tempResourceId)) { if (customAttributesByField.isEmpty()) { return; } @@ -1132,9 +1168,6 @@ public void putCustomMetadataAttributes(final String tempResourceId, .build()), customAttributes); }catch (Exception e){ - if (AssetStorageFeature.isEnabled()) { - throw new DotDataException("Unable to save temporary binary metadata for " + tempResourceId, e); - } Logger.error(FileMetadataAPIImpl.class, "Error saving custom attributes", e); } } @@ -1142,9 +1175,13 @@ public void putCustomMetadataAttributes(final String tempResourceId, /** * {@inheritDoc} - * @param tempResourceId - * @return - * @throws DotDataException + * + *

With S3 asset storage on, the metadata is read as described in + * {@link #temporaryMetadata(String)}, bypassing node caches. + * + * @param tempResourceId the temporary resource id + * @return the metadata, or empty if there is none + * @throws DotDataException if the metadata cannot be read */ public Optional getMetadata(final String tempResourceId) throws DotDataException { diff --git a/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java b/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java index ffc49787152e..1014128cca92 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java +++ b/dotCMS/src/main/java/com/dotcms/storage/TemporaryAssetStorage.java @@ -4,12 +4,14 @@ import com.dotmarketing.exception.DotDataException; import com.dotmarketing.util.Config; import com.dotmarketing.util.ConfigUtils; +import com.dotmarketing.util.Logger; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; import java.io.Serializable; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -207,29 +209,47 @@ private void removeMetadataAndRenditions(final String id) throws DotDataExceptio APILocator.getBinaryAssetStorageAPI().deleteGeneratedFiles(id); } - /** Timestamped payload prefixes also expose interrupted uploads after their receipt is gone. */ + /** + * Removes the S3 objects of temporary uploads whose receipt has expired: the receipt, the + * payload, the shared metadata and any generated renditions. Timestamped payload prefixes also + * expose interrupted uploads after their receipt is gone, and those are removed too. + * + *

Local copies are left in place. The existing {@code CLEANUP_TMP_FILES_OLDER_THAN_HOURS} age + * rule in {@code BinaryCleanupJob} removes them, as it does with the flag off, so a check-in that + * resolved the file just before expiry keeps its source. The local upload marker stops an + * expired local copy from being served once its receipt is gone. + * + *

Each upload is cleaned independently. A failure is logged, that upload keeps its receipt and + * payload for the next run, and the remaining uploads are still processed. One combined failure + * is thrown at the end so the scheduled job still reports it. + * + * @throws DotDataException if S3 cannot be listed, or if any upload could not be cleaned + */ public void cleanupExpired() throws DotDataException { if (!AssetStorageFeature.isEnabled()) return; + final List failures = new ArrayList<>(); + final var retained = new HashSet(); for (String path : remote().listObjectPaths(GROUP, RECEIPTS)) { if (!path.startsWith(RECEIPTS) || !path.endsWith(".json")) continue; final String id = path.substring(RECEIPTS.length(), path.length() - ".json".length()); - final var record = receipt(id); - if (record.isEmpty() || !record.get().expired()) continue; - removeMetadataAndRenditions(id); - delete(record.get().prefix()); - delete(path); try { - final Path directory = file(id, "placeholder").toPath().getParent(); - com.liferay.util.FileUtil.deltree(directory.toFile()); - if (Files.exists(directory)) throw new IOException("Temporary cache directory remains"); - } catch (IOException failure) { - throw new DotDataException("Unable to remove expired temporary cache " + id, failure); + final var record = receipt(id); + if (record.isEmpty() || !record.get().expired()) continue; + removeMetadataAndRenditions(id); + delete(record.get().prefix()); + delete(path); + } catch (DotDataException | RuntimeException failure) { + // Keep this upload's payload with its receipt, so the next run can finish it. + retained.add(id); + failures.add(failure); + Logger.warn(this, "Unable to clean expired temporary upload " + id + "; next cleanup will retry: " + + failure.getMessage()); } } final var expired = new HashSet(); for (String path : remote().listObjectPaths(GROUP, "data/")) { final String[] parts = path.split("/", 4); - if (parts.length != 4 || !parts[0].equals("data") || !validId(parts[2])) continue; + if (parts.length != 4 || !parts[0].equals("data") || !validId(parts[2]) || retained.contains(parts[2])) continue; try { final var record = new Receipt(parts[2], parts[3], Long.parseLong(parts[1]), List.of()); if (record.expired()) expired.add(record.prefix()); @@ -237,7 +257,21 @@ final var record = new Receipt(parts[2], parts[3], Long.parseLong(parts[1]), Lis // Only this uploader's timestamped namespace belongs to this cleanup. } } - for (String prefix : expired) delete(prefix); + for (String prefix : expired) { + try { + delete(prefix); + } catch (DotDataException | RuntimeException failure) { + failures.add(failure); + Logger.warn(this, "Unable to remove expired temporary payload " + prefix + "; next cleanup will retry: " + + failure.getMessage()); + } + } + if (!failures.isEmpty()) { + final var combined = new DotDataException("Unable to clean " + failures.size() + + " expired temporary upload object(s)", failures.get(0)); + failures.stream().skip(1).forEach(combined::addSuppressed); + throw combined; + } } private void delete(final String path) throws DotDataException { diff --git a/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java b/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java index 2ade41535464..636bf862c68a 100644 --- a/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java +++ b/dotCMS/src/main/java/com/dotcms/storage/WebdavTemporaryStorage.java @@ -151,24 +151,53 @@ public Entry stat(File file) { } } + /** + * Lists the live direct children of a staging directory, sorted by path. + * + *

The S3 listing returns every record in the subtree, so records are grouped by key first and + * only what decides each direct child is read: the child's own record, and only when that record + * is missing, deleted or expired, its descendants until one live descendant shows that the child + * still exists as an implicit directory. Deeper records under a live child are never fetched. + * Records that disappear between the listing and the read are skipped. + * + * @param directory the local path of the staging directory + * @return the live files and directories directly inside it + * @throws IOException if the directory or a listed key is not a valid staging path + * @throws DotDataException if S3 cannot be read + */ public List children(File directory) throws IOException, DotDataException { - final String prefix = key(directory) + "/"; - final Map children = new java.util.TreeMap<>(); - for (String object : remote().listObjectPaths(GROUP, recordPrefix(key(directory)))) { + final String parent = key(directory); + final String prefix = parent + "/"; + // Direct child path to the descendant record paths below it, in listing order. + final Map> descendants = new java.util.TreeMap<>(); + final var recorded = new java.util.HashSet(); + for (String object : remote().listObjectPaths(GROUP, recordPrefix(parent))) { final String path = recordPath(object); - if (path.equals(key(directory))) continue; + if (path.equals(parent)) continue; if (!path.startsWith(prefix)) throw new IOException("Invalid staging listing"); - final Entry entry = read(path); - if (entry == null || entry.expired()) continue; final String suffix = path.substring(prefix.length()); final int slash = suffix.indexOf('/'); - if (slash < 0) children.put(path, entry); - else { - final String parent = prefix + suffix.substring(0, slash); - children.putIfAbsent(parent, new Entry(parent, true, 0, entry.modified(), "")); + final String child = slash < 0 ? path : prefix + suffix.substring(0, slash); + final List nested = descendants.computeIfAbsent(child, ignored -> new ArrayList<>()); + if (slash < 0) recorded.add(path); + else nested.add(path); + } + final List children = new ArrayList<>(); + for (var child : descendants.entrySet()) { + final Entry own = recorded.contains(child.getKey()) ? read(child.getKey()) : null; + if (own != null && !own.expired()) { + children.add(own); + continue; + } + for (String path : child.getValue()) { + final Entry entry = read(path); + if (entry != null && !entry.expired()) { + children.add(new Entry(child.getKey(), true, 0, entry.modified(), "")); + break; + } } } - return new ArrayList<>(children.values()); + return children; } private String publish(String path, Entry entry, String pending, String expected) throws DotDataException { diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java b/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java index 6f1ee1be9ecf..d8d6fd9dba04 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/DotWebdavHelper.java @@ -412,7 +412,20 @@ private String temporaryPath(String path) { return String.join("/", parts); } - List temporaryChildren(String url, boolean autoPublish) throws IOException { + /** + * Lists the WebDAV temporary files and folders stored in S3 directly under a CMS folder, site + * root or language folder, for use with S3 asset storage on. + * + *

Each file resource is built from the entry the listing already read, so a file that a + * client deletes while the folder is being listed cannot fail the listing. If S3 cannot be + * read, the failure is logged and only the temporary children are left out, so the CMS files + * and folders around them still list. + * + * @param url the WebDAV URL of the containing folder + * @param autoPublish whether the resources are served from an auto-publishing mount + * @return the temporary children, or an empty list if they could not be read + */ + List temporaryChildren(String url, boolean autoPublish) { final var staging = com.dotcms.storage.WebdavTemporaryStorage.getInstance(); final List result = new ArrayList<>(); final String prefix = url.endsWith("/") ? url : url + "/"; @@ -422,10 +435,12 @@ List temporaryChildren(String url, boolean autoPublish) throws IOExcep final String child = prefix + file.getName(); if (!isTempResource(child)) continue; result.add(entry.directory() ? new TempFolderResourceImpl(child, file, autoPublish) - : new TempFileResourceImpl(file, child, autoPublish)); + : new TempFileResourceImpl(file, child, autoPublish, entry)); } - } catch (DotDataException failure) { - throw new IOException("Unable to list WebDAV staging resources", failure); + } catch (IOException | DotDataException | RuntimeException failure) { + Logger.warn(this, "Unable to list WebDAV temporary files under " + stripLogControls(url) + + "; leaving them out of the listing: " + failure.getMessage()); + return new ArrayList<>(); } return result; } @@ -695,11 +710,24 @@ public void copyTempFileToStorage(File fromFile, String destPath,User user,boole setResourceContent(destPath, in, null, null, new Date(fromFile.lastModified()),user, autoPublish); } + /** + * Copies a CMS file to another WebDAV path. The copy is always saved as an unpublished working + * version, whatever mount the request came through, so it needs only edit permission. The + * {@code autoPublish} argument is not used for that reason. + * + *

With S3 asset storage on, the source stream is closed once the copy is saved. + * + * @param fromPath the WebDAV path of the source file + * @param toPath the WebDAV path of the copy + * @param user the user making the copy + * @param autoPublish whether the request came through an auto-publishing mount (ignored) + * @throws Exception if the source cannot be read or the copy cannot be saved + */ public void copyResource(String fromPath, String toPath, User user, boolean autoPublish) throws Exception { if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { try (InputStream input = getResourceContent(fromPath, user)) { if (input == null) throw new IOException("Missing WebDAV copy source"); - setResourceContent(toPath, input, null, null, new Date(), user, autoPublish); + setResourceContent(toPath, input, null, null, new Date(), user, false); } return; } diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java index 7bff0c1a9d10..49113a454d90 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/HostResourceImpl.java @@ -111,6 +111,14 @@ public int compareTo(Object o) { return 0; } + /** + * Finds a folder, file or temporary file directly under the site root. With S3 asset storage + * on, a temporary name is looked up among the site's S3 temporary children, which are left out + * (so the lookup finds nothing) if S3 cannot be read. + * + * @param childName the name of the child + * @return the child, or {@code null} if there is none + */ public Resource child(String childName) { if (dotDavHelper.isSameTargetAndDestinationResourceOnMove(childName)) { //This a small hack that prevents Milton's MoveHandler from removing the destination folder when the source and destination are the same. @@ -118,12 +126,8 @@ public Resource child(String childName) { } final User user = (User)HttpManager.request().getAuthorization().getTag(); if (com.dotcms.storage.AssetStorageFeature.isEnabled() && dotDavHelper.isTempResource(childName)) { - try { - return dotDavHelper.temporaryChildren(path, isAutoPub).stream() - .filter(resource -> resource.getName().equals(childName)).findFirst().orElse(null); - } catch (IOException failure) { - throw new DotRuntimeException("Unable to find WebDAV staging child", failure); - } + return dotDavHelper.temporaryChildren(path, isAutoPub).stream() + .filter(resource -> resource.getName().equals(childName)).findFirst().orElse(null); } final String uri="/"+childName; @@ -150,6 +154,12 @@ else if(ident.getAssetType().equals("contentlet")) { return null; } + /** + * Lists the folders, files and temporary files directly under the site root. With S3 asset + * storage on, the temporary files come from S3 and are left out if S3 cannot be read. + * + * @return the children of the site root + */ public List getChildren() { User user=(User)HttpManager.request().getAuthorization().getTag(); List folders = listFolders(); @@ -210,11 +220,7 @@ public List getChildren() { } java.io.File tempDir = new java.io.File(dotDavHelper.getTempDir().getPath() + java.io.File.separator + host.getHostname()); if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { - try { - frs.addAll(dotDavHelper.temporaryChildren(prePath + host.getHostname(), isAutoPub)); - } catch (IOException failure) { - throw new DotRuntimeException("Unable to list WebDAV staging resources", failure); - } + frs.addAll(dotDavHelper.temporaryChildren(prePath + host.getHostname(), isAutoPub)); } else if(tempDir.exists() && tempDir.isDirectory()){ java.io.File[] files = tempDir.listFiles(); for (java.io.File file : files) { diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java index c2ac9c632824..5de97e0ea8e4 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/LanguageFolderResourceImpl.java @@ -124,7 +124,11 @@ public Resource child(String childName) { return null; } - /* (non-Javadoc) + /** + * Lists the language files and temporary files in this language folder. With S3 asset storage + * on, the temporary files come from S3 and are left out if S3 cannot be read. + * + * @return the children of this folder * @see com.bradmcevoy.http.CollectionResource#getChildren() */ public List getChildren() { @@ -149,12 +153,8 @@ public List getChildren() { if (com.dotcms.storage.AssetStorageFeature.isEnabled()) { final String prefix = Config.getBooleanProperty("WEBDAV_LEGACY_PATHING", false) ? "/webdav/autopub/" : "/webdav/live/" + dotDavHelper.getLanguage() + "/"; - try { - result.addAll(dotDavHelper.temporaryChildren(prefix + "system/languages/" + path, true)); - return result; - } catch (IOException failure) { - throw new com.dotmarketing.exception.DotRuntimeException("Unable to list WebDAV staging resources", failure); - } + result.addAll(dotDavHelper.temporaryChildren(prefix + "system/languages/" + path, true)); + return result; } File f = new File(tempDir.getPath() + File.separator + "system" + File.separator + "languages" + path); File[] c = f.listFiles(); diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java index 9d2b289ff0d2..27207c9b8ec5 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/ResourceFactoryImpl.java @@ -46,6 +46,17 @@ public Resource getResource(String davHost, String url) { return getResource(davHost, url, dotDavHelper, hostAPI); } + /** + * Resolves a WebDAV URL to the site, folder, file, language file or temporary file it names. + * With S3 asset storage on, a temporary file is resolved from its S3 entry, and that entry is + * reused to build the resource rather than looked up a second time. + * + * @param davHost the host header of the request + * @param url the requested WebDAV URL + * @param dotDavHelper the WebDAV helper + * @param hostAPI the site API + * @return the resource, or {@code null} if the URL names nothing + */ public static Resource getResource(final String davHost, String url, final DotWebdavHelper dotDavHelper, final HostAPI hostAPI) { if (!com.dotcms.storage.AssetStorageFeature.isEnabled() || !dotDavHelper.isTempResource(url)) url = url.toLowerCase(); Logger.debug(ResourceFactoryImpl.class, "WebDav ResourceFactory: Host is " + davHost + " and the url is " + url); @@ -119,7 +130,7 @@ public static Resource getResource(final String davHost, String url, final DotWe final var entry = com.dotcms.storage.WebdavTemporaryStorage.getInstance().stat(tempFile); if (entry == null) return null; return entry.directory() ? new TempFolderResourceImpl(url, tempFile, dotDavHelper.isAutoPub(url)) - : new TempFileResourceImpl(tempFile, url, dotDavHelper.isAutoPub(url)); + : new TempFileResourceImpl(tempFile, url, dotDavHelper.isAutoPub(url), entry); } if(tempFile == null || !tempFile.exists()){ return null; diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java index e97fd2618b01..b43d59b29cda 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFileResourceImpl.java @@ -42,12 +42,33 @@ public class TempFileResourceImpl implements FileResource, LockableResource { private boolean isAutoPub = false; private PermissionAPI perAPI; + /** + * Creates a resource for a WebDAV temporary file. With S3 asset storage on, the file's entry is + * looked up in S3 first. + * + * @param file the local path of the temporary file + * @param url the WebDAV URL of the file + * @param isAutoPub whether the file is served from an auto-publishing mount + * @throws IllegalArgumentException if the file is a directory, or with the flag on, if it does + * not exist + */ public TempFileResourceImpl(File file, String url, boolean isAutoPub) { - entry = com.dotcms.storage.AssetStorageFeature.isEnabled() - ? com.dotcms.storage.WebdavTemporaryStorage.getInstance().stat(file) : null; - if (com.dotcms.storage.AssetStorageFeature.isEnabled() && entry == null) { - throw new IllegalArgumentException("Missing WebDAV staging file: " + url); - } + this(file, url, isAutoPub, storedEntry(file, url)); + } + + /** + * Creates a resource from an entry that a listing has already read, so no second lookup is + * needed and a file deleted in the meantime cannot fail the listing. + * + * @param file the local path of the temporary file + * @param url the WebDAV URL of the file + * @param isAutoPub whether the file is served from an auto-publishing mount + * @param entry the S3 entry of the file, or {@code null} with the flag off + * @throws IllegalArgumentException if the file is a directory + */ + TempFileResourceImpl(File file, String url, boolean isAutoPub, + com.dotcms.storage.WebdavTemporaryStorage.Entry entry) { + this.entry = entry; if( entry != null ? entry.directory() : file.isDirectory() ){ Logger.error(this, "Trying to get a temp file which is actually a directory!!!"); throw new IllegalArgumentException("Static resource must be a file, this is a directory: " + file.getAbsolutePath()); @@ -60,6 +81,21 @@ public TempFileResourceImpl(File file, String url, boolean isAutoPub) { } + /** + * Looks up the S3 entry of a temporary file when S3 asset storage is on. + * + * @param file the local path of the temporary file + * @param url the WebDAV URL of the file, used in the error message + * @return the entry, or {@code null} with the flag off + * @throws IllegalArgumentException with the flag on, if the file does not exist + */ + private static com.dotcms.storage.WebdavTemporaryStorage.Entry storedEntry(File file, String url) { + if (!com.dotcms.storage.AssetStorageFeature.isEnabled()) return null; + final var entry = com.dotcms.storage.WebdavTemporaryStorage.getInstance().stat(file); + if (entry == null) throw new IllegalArgumentException("Missing WebDAV staging file: " + url); + return entry; + } + public String getUniqueId() { return entry == null ? file.hashCode() + "" : "webdav-temporary:" + entry.path(); } diff --git a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java index c7588dddf9a0..d06411ebc41d 100644 --- a/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java +++ b/dotCMS/src/main/java/com/dotmarketing/webdav/TempFolderResourceImpl.java @@ -85,7 +85,11 @@ public Resource child(String childName) { return null; } - /* (non-Javadoc) + /** + * Lists the files and folders in this temporary folder. With S3 asset storage on, they come + * from S3, and each file resource is built from the entry the listing already read. + * + * @return the children of this folder * @see com.bradmcevoy.http.CollectionResource#getChildren() */ public List getChildren() { @@ -96,7 +100,7 @@ public List getChildren() { for (var entry : storage.children(folder)) { final File file = storage.file(entry.path()); result.add(entry.directory() ? new TempFolderResourceImpl(file.getPath(), file, isAutoPub) - : new TempFileResourceImpl(file, file.getPath(), isAutoPub)); + : new TempFileResourceImpl(file, file.getPath(), isAutoPub, entry)); } return result; } catch (Exception failure) { diff --git a/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java b/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java index 1ca248a6e204..0471a90767ef 100644 --- a/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java +++ b/dotCMS/src/test/java/com/dotcms/storage/TemporaryAssetStorageTest.java @@ -60,7 +60,10 @@ static void assertRoundTrip(final StoragePersistenceAPI remote, final Path root) nodeB.cleanupExpired(); assertFalse(remote.existsObject(TemporaryAssetStorage.GROUP, receipt.key())); assertTrue(nodeB.receipt(id).isEmpty()); - assertFalse(restored.exists()); + // The local copy is left to the CLEANUP_TMP_FILES_OLDER_THAN_HOURS age rule, so a + // check-in that resolved it just before expiry keeps its source. + assertTrue(restored.exists(), "Expiry must not delete local bytes before the local age rule does"); + assertTrue(nodeB.isManagedLocally(id), "The marker keeps the expired local copy from legacy access"); verify(binaries).deleteGeneratedFiles(id); } finally { Config.setProperty(AssetStorageFeature.FLAG, flag); @@ -111,6 +114,49 @@ void rejectedUploadRetainsLocalBytesAndDisabledStorageDoesNoRemoteWork() throws } } + /** One unreadable receipt must not stop the cleanup of the other expired uploads. */ + @Test + void cleanupContinuesPastAnUnreadableReceiptAndStillReportsIt() throws Exception { + final String flag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + final String ttl = Config.getStringProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", null); + try (var locator = mockStatic(APILocator.class)) { + Config.setProperty(AssetStorageFeature.FLAG, true); + Config.setProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", 1800); + final var metadata = mock(FileStorageAPI.class); + final var binaries = mock(BinaryAssetStorageAPI.class); + locator.when(APILocator::getFileStorageAPI).thenReturn(metadata); + locator.when(APILocator::getBinaryAssetStorageAPI).thenReturn(binaries); + final var remote = memoryRemote(); + final var store = new TemporaryAssetStorage(remote, root); + // Listed first, and its receipt cannot be parsed. + remote.backfillObject(TemporaryAssetStorage.GROUP, ".receipts/temp_Aaa-Broken.json", null, null, + new HashMap<>(Map.of("name", 42))); + final String brokenPayload = "data/1/temp_Aaa-Broken/Broken.GIF"; + remote.backfillFile(TemporaryAssetStorage.GROUP, brokenPayload, + Files.writeString(root.resolve("broken.tmp"), "broken").toFile()); + final String id = "temp_Zzz-Valid"; + final File source = store.file(id, "Valid.GIF"); + Files.createDirectories(source.toPath().getParent()); + Files.writeString(source.toPath(), "valid upload"); + Files.writeString(source.toPath().resolveSibling(TemporaryAssetStorage.PERMISSIONS_FILE), "[\"owner\"]"); + store.store(id, source); + final var receipt = store.receipt(id).orElseThrow(); + Config.setProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", 0); + + assertThrows(DotDataException.class, store::cleanupExpired, "The job must still report the failure"); + + assertTrue(store.receipt(id).isEmpty(), "The valid expired upload is cleaned despite the broken one"); + assertFalse(remote.existsObject(TemporaryAssetStorage.GROUP, receipt.key())); + verify(binaries).deleteGeneratedFiles(id); + assertTrue(remote.existsObject(TemporaryAssetStorage.GROUP, ".receipts/temp_Aaa-Broken.json")); + assertTrue(remote.existsObject(TemporaryAssetStorage.GROUP, brokenPayload), + "A failed upload keeps its payload with its receipt for the next run"); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, flag); + Config.setProperty("TEMP_RESOURCE_MAX_AGE_SECONDS", ttl); + } + } + private StoragePersistenceAPI memoryRemote() throws Exception { final var remote = mock(StoragePersistenceAPI.class); final Map objects = new HashMap<>(); @@ -132,7 +178,7 @@ private StoragePersistenceAPI memoryRemote() throws Exception { }); doAnswer(call -> { Files.delete(call.getArgument(0).toPath()); return null; }).when(remote).releaseRetrievedFile(any()); when(remote.listObjectPaths(anyString(), anyString())).thenAnswer(call -> - objects.keySet().stream().filter(key -> key.startsWith(call.getArgument(1))).toList()); + objects.keySet().stream().filter(key -> key.startsWith(call.getArgument(1))).sorted().toList()); when(remote.deleteObjectAndReferences(anyString(), anyString())).thenAnswer(call -> { final String prefix = call.getArgument(1); objects.remove(prefix); diff --git a/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java b/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java index ed12701f23be..435f7bdd877b 100644 --- a/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java +++ b/dotCMS/src/test/java/com/dotcms/storage/TemporaryMetadataStorageTest.java @@ -69,6 +69,41 @@ void portableMetadataPreservesLegacyValuesAndClearedFocalPointsAcrossNodes() thr } } + /** + * FocalPointImageFilter names existing content as {@code temp_}. Such an id never gets an + * upload receipt, so its metadata must stay in the local temporary directory that + * BinaryCleanupJob ages out, never in S3 where nothing would remove it. + */ + @Test + void inodeBasedFocalPointIdsStayInTheLocallyCleanedDirectory() throws Exception { + final String oldFlag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + final String id = "temp_" + java.util.UUID.randomUUID(); + final String localPath = "/node-a/assets/tmp_upload/" + id + "/" + id + FileMetadataAPI.META_TMP; + final var files = mock(FileStorageAPI.class); + try (var locator = mockStatic(APILocator.class); var caches = mockStatic(CacheLocator.class); + var paths = mockStatic(ConfigUtils.class)) { + Config.setProperty(AssetStorageFeature.FLAG, true); + locator.when(APILocator::getFileStorageAPI).thenReturn(files); + caches.when(CacheLocator::getMetadataCache).thenReturn(mock(MetadataCache.class)); + paths.when(ConfigUtils::getAssetTempPath).thenReturn("/node-a/assets/tmp_upload"); + final var api = new FileMetadataAPIImpl(); + + api.putCustomMetadataAttributes(id, Map.of("HeroImage", Map.of("focalPoint", "0.5,0.5"))); + verify(files).putCustomMetadataAttributes(argThat(request -> + request.getStorageKey().getStorage() == StorageType.FILE_SYSTEM + && request.getStorageKey().getPath().equals(localPath)), eq(Map.of("focalPoint", "0.5,0.5"))); + verify(files, never()).setMetadata(any(), any()); + + when(files.retrieveRawMetaData(any())).thenReturn(Map.of("dot:focalPoint", "0.5,0.5")); + assertEquals("0.5,0.5", api.getMetadata(id).orElseThrow().getCustomMeta().get("focalPoint")); + verify(files).retrieveRawMetaData(argThat(key -> key.getStorage() == StorageType.FILE_SYSTEM + && key.getPath().equals(localPath))); + verify(files, never()).retrieveRawMetaData(argThat(key -> key.getStorage() != StorageType.FILE_SYSTEM)); + } finally { + Config.setProperty(AssetStorageFeature.FLAG, oldFlag); + } + } + @Test void disabledWritesRetainFilesystemPathAndProvider() throws Exception { final String oldFlag = Config.getStringProperty(AssetStorageFeature.FLAG, null); diff --git a/dotCMS/src/test/java/com/dotmarketing/webdav/WebdavTemporaryStorageTest.java b/dotCMS/src/test/java/com/dotmarketing/webdav/WebdavTemporaryStorageTest.java new file mode 100644 index 000000000000..ca54dbc691c8 --- /dev/null +++ b/dotCMS/src/test/java/com/dotmarketing/webdav/WebdavTemporaryStorageTest.java @@ -0,0 +1,154 @@ +package com.dotmarketing.webdav; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import com.bradmcevoy.http.Resource; +import com.dotcms.storage.AssetStorageFeature; +import com.dotcms.storage.ObjectSnapshot; +import com.dotcms.storage.StoragePersistenceAPI; +import com.dotcms.storage.WebdavTemporaryStorage; +import com.dotmarketing.business.APILocator; +import com.dotmarketing.business.CacheLocator; +import com.dotmarketing.exception.DotDataException; +import com.dotmarketing.portlets.languagesmanager.business.LanguageAPI; +import com.dotmarketing.portlets.languagesmanager.model.Language; +import com.dotmarketing.util.Config; +import com.dotmarketing.util.ConfigUtils; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Runs the S3-backed WebDAV temporary-file storage against an in-memory remote that models + * conditional writes, so its listing behavior is covered without MinIO. + */ +class WebdavTemporaryStorageTest { + @TempDir Path root; + private String previousFlag; + private final Map objects = new TreeMap<>(); + private final List reads = new ArrayList<>(); + + @BeforeEach + void enableFlag() { + previousFlag = Config.getStringProperty(AssetStorageFeature.FLAG, null); + Config.setProperty(AssetStorageFeature.FLAG, true); + } + + @AfterEach + void restoreFlag() { + Config.setProperty(AssetStorageFeature.FLAG, previousFlag); + } + + /** Listing a folder reads each direct child's own record, and descendants only for an implicit folder. */ + @Test + void childrenReadsOnlyTheRecordsThatDecideEachDirectChild() throws Exception { + final var remote = memoryRemote(); + final var storage = new WebdavTemporaryStorage(remote, root); + storage.mkdir(file("example.com/folder/.Trash")); + storage.store(file("example.com/folder/.Trash/a.txt"), payload("a")); + storage.store(file("example.com/folder/.Trash/deep/b.txt"), payload("b")); + storage.store(file("example.com/folder/._Hero.GIF"), payload("hero")); + storage.store(file("example.com/folder/(implicit)/c.txt"), payload("c")); + storage.store(file("example.com/folder/._Gone"), payload("gone")); + storage.delete(file("example.com/folder/._Gone")); + storage.store(file("example.com/other/._Unrelated"), payload("other")); + reads.clear(); + + final var children = storage.children(file("example.com/folder")); + + assertEquals(List.of("example.com/folder/(implicit)", "example.com/folder/.Trash", "example.com/folder/._Hero.GIF"), + children.stream().map(WebdavTemporaryStorage.Entry::path).toList()); + assertEquals(List.of(true, true, false), children.stream().map(WebdavTemporaryStorage.Entry::directory).toList()); + assertEquals(4, children.get(2).size()); + // .Trash, ._Hero.GIF and the ._Gone tombstone have their own records. The implicit folder + // has none, so one live descendant is read to show it exists. + assertEquals(4, reads.size(), "Unexpected record reads: " + reads); + assertTrue(reads.stream().noneMatch(key -> key.contains("Trash/a") || key.contains("Trash/deep")), + "Records below a live folder must not be read: " + reads); + assertTrue(reads.stream().noneMatch(key -> key.contains("other")), "Unrelated records must not be read"); + } + + /** A temporary file deleted mid-listing, or an S3 failure, must not fail the containing folder listing. */ + @Test + void temporaryChildrenSurviveVanishingFilesAndLeaveOutUnreadableOnes() throws Exception { + final var remote = memoryRemote(); + final var storage = new WebdavTemporaryStorage(remote, root); + storage.store(file("example.com/folder/._Hero.GIF"), payload("hero")); + storage.store(file("example.com/folder/._Vanished"), payload("vanished")); + // ._Vanished is deleted between the listing and its read. ._Hero.GIF is deleted right after + // its first read, which is when a second lookup per file used to fail the whole listing. + reads.clear(); + doAnswer(call -> { + final String key = call.getArgument(1); + final boolean readBefore = reads.contains(key); + reads.add(key); + return key.contains("Vanished") || (key.contains("Hero") && readBefore) ? null : objects.get(key); + }).when(remote).readObjectSnapshot(anyString(), anyString(), any()); + + try (var locator = mockStatic(APILocator.class); var caches = mockStatic(CacheLocator.class); + var paths = mockStatic(ConfigUtils.class, CALLS_REAL_METHODS); + var singleton = mockStatic(WebdavTemporaryStorage.class)) { + final var language = mock(LanguageAPI.class); + when(language.getDefaultLanguage()).thenReturn(new Language()); + locator.when(APILocator::getLanguageAPI).thenReturn(language); + paths.when(ConfigUtils::getAssetTempPath).thenReturn(root.toString()); + singleton.when(WebdavTemporaryStorage::getInstance).thenReturn(storage); + final var helper = new DotWebdavHelper(); + + final List listed = helper.temporaryChildren("/example.com/folder", false); + assertEquals(List.of("._Hero.GIF"), listed.stream().map(Resource::getName).toList()); + assertEquals(4L, ((TempFileResourceImpl) listed.get(0)).getContentLength()); + + doThrow(new DotDataException("S3 unavailable")).when(remote).listObjectPaths(anyString(), anyString()); + assertTrue(helper.temporaryChildren("/example.com/folder", false).isEmpty(), + "An S3 failure leaves out only the temporary children"); + } + } + + private File file(final String key) { + return root.resolve(key).toFile(); + } + + private File payload(final String content) throws Exception { + return Files.writeString(Files.createTempFile(root, "payload-", ".tmp"), content).toFile(); + } + + /** A remote whose records carry versions, so conditional writes and tombstones behave as on S3. */ + private StoragePersistenceAPI memoryRemote() throws Exception { + final var remote = mock(StoragePersistenceAPI.class); + when(remote.listObjectPaths(anyString(), anyString())).thenAnswer(call -> objects.keySet().stream() + .filter(key -> key.startsWith(call.getArgument(1))).toList()); + when(remote.readObjectSnapshot(anyString(), anyString(), any())).thenAnswer(call -> { + reads.add(call.getArgument(1)); + return objects.get(call.getArgument(1)); + }); + when(remote.writeObjectIfMatch(anyString(), anyString(), any(), any())).thenAnswer(call -> { + final String key = call.getArgument(1); + final ObjectSnapshot current = objects.get(key); + if (!Objects.equals(current == null ? null : current.version(), call.getArgument(3))) return null; + final String version = UUID.randomUUID().toString(); + objects.put(key, new ObjectSnapshot(key, call.getArgument(2), version, System.currentTimeMillis())); + return version; + }); + when(remote.backfillFile(anyString(), anyString(), any())).thenAnswer(call -> { + final String key = call.getArgument(1); + objects.put(key, new ObjectSnapshot(key, Files.readString(call.getArgument(2).toPath()), + UUID.randomUUID().toString(), System.currentTimeMillis())); + return true; + }); + when(remote.deleteObjectAndReferences(anyString(), anyString())).thenAnswer(call -> + objects.remove(call.getArgument(1)) != null); + return remote; + } +} diff --git a/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java b/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java index 03f99f3342a3..3fac2c298ea7 100644 --- a/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java +++ b/dotcms-integration/src/test/java/com/dotmarketing/webdav/DotWebdavHelperTest.java @@ -207,7 +207,6 @@ private static boolean evictEventually(final com.dotcms.storage.binary.BinaryAss @Test public void uploadColdReadCopyAndOverwriteRetainHistoricalBytes() throws Exception { final boolean enabled = com.dotcms.storage.AssetStorageFeature.isEnabled(); - Assert.assertEquals(Boolean.getBoolean("s3.cms.enabled"), enabled); final Host host = APILocator.getHostAPI().findDefaultHost(APILocator.systemUser(), false); final Folder folder = new FolderDataGen().site(host).nextPersisted(); final var user = APILocator.systemUser(); @@ -237,7 +236,9 @@ public void uploadColdReadCopyAndOverwriteRetainHistoricalBytes() throws Excepti helper.copyResource(source, base + "Copy-Mixed.TXT", user, enabled); final var copy = (com.dotmarketing.portlets.contentlet.model.Contentlet) helper.loadFile(base + "Copy-Mixed.TXT", user); Assert.assertNotNull(copy); - if (enabled) Assert.assertTrue(copy.isLive()); + // A WebDAV copy is an unpublished working version in both modes, even when requested + // with auto-publish, so it needs only edit permission. + Assert.assertFalse(copy.isLive()); final var copyFile = copy.getBinary(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD); if (enabled) Assert.assertTrue(evictEventually(binaries, copyFile)); try (var input = copy.getBinaryStream(com.dotmarketing.portlets.fileassets.business.FileAssetAPI.BINARY_FIELD)) {