From e721738ed364a671f7c7c03b1e7ff2e2b41f6128 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 1 Jun 2026 16:27:04 +0800 Subject: [PATCH 1/4] HDDS-15447. Persist bucket scanned key pointer periodically --- .../src/main/resources/ozone-default.xml | 15 + .../content/design/lifecycle-task-resume.md | 80 + hadoop-hdds/docs/content/feature/Lifecycle.md | 11 +- .../docs/content/feature/Lifecycle.zh.md | 11 +- .../java/org/apache/hadoop/ozone/OmUtils.java | 2 + .../apache/hadoop/ozone/om/OMConfigKeys.java | 8 + .../hadoop/ozone/om/helpers/OmLCRule.java | 13 +- .../om/helpers/OmLifecycleScanState.java | 255 ++++ .../hadoop/ozone/om/helpers/TestOmLCRule.java | 33 + .../om/helpers/TestOmLifecycleScanState.java | 79 + .../src/main/proto/OmClientProtocol.proto | 23 + .../hadoop/ozone/om/OMMetadataManager.java | 8 + .../ozone/om/OmMetadataManagerImpl.java | 8 + .../hadoop/ozone/om/codec/OMDBDefinition.java | 11 +- .../ratis/utils/OzoneManagerRatisUtils.java | 3 + .../om/request/key/OMKeysDeleteRequest.java | 29 +- .../key/OmKeysDeleteRequestWithFSO.java | 6 +- .../OMLifecycleSaveScanStateRequest.java | 72 + .../om/response/key/OMKeysDeleteResponse.java | 13 +- .../key/OMKeysDeleteResponseWithFSO.java | 16 +- .../OMLifecycleSaveScanStateResponse.java | 51 + .../ozone/om/service/KeyLifecycleService.java | 1046 ++++++++++--- .../ozone/om/TestOmMetadataManager.java | 4 +- .../TestOMLifecycleSaveScanStateRequest.java | 119 ++ .../key/TestOMKeysDeleteResponse.java | 2 +- .../key/TestOMKeysDeleteResponseWithFSO.java | 2 +- .../TestOMLifecycleSaveScanStateResponse.java | 61 + .../om/service/TestKeyLifecycleService.java | 1289 ++++++++++++++--- 28 files changed, 2865 insertions(+), 405 deletions(-) create mode 100644 hadoop-hdds/docs/content/design/lifecycle-task-resume.md create mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java create mode 100644 hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index a7c4d56e5312..2dbc98adf1ea 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -5140,4 +5140,19 @@ lifecycle rules. + + ozone.lifecycle.service.state.save.interval.ms + 300000 + OZONE + The interval of bucket scan task saves its pointer to DB. Default is 5 mins. + + + ozone.lifecycle.service.state.save.keys.processed + 100000 + OZONE + Bucket scan task will save its pointer to DB by default every 100000 keys are scanned. Bucket scan + pointer save will happen when either ozone.lifecycle.service.state.save.interval.ms or + ozone.lifecycle.service.state.save.keys.processed is satisfied. + + diff --git a/hadoop-hdds/docs/content/design/lifecycle-task-resume.md b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md new file mode 100644 index 000000000000..0898b19f813b --- /dev/null +++ b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md @@ -0,0 +1,80 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +# Design for Resumable Lifecycle Scans(HDDS-8342) + +## Problem Statement: + +The `HDDS-8342` branch introduces the `KeyLifecycleService`, a background service running on the Ozone Manager (OM) Leader to enforce bucket lifecycle rules (expiration, moving to trash, and aborting incomplete multipart uploads). +The entire bucket is scanned in a single `call()` execution. If the OM restarts, crashes, or a leader transfer occurs, the scan state is lost. The new leader must restart the scan from the beginning. +For buckets with billions of keys, the scan may never complete if leader transfers happen periodically. + +## Design: Persisting Bucket Scan Pointers + +To solve the resumability issues, we need to persist the scan progress (the "pointer") to the OM DB. This ensures that a new OM leader can resume from where the previous leader left off. + +### 2.1 Data Structure for the Scan Pointer + +Define a new Protobuf message `LifecycleScanState` to capture the exact position of the scan. + +```protobuf +message LifecycleScanState { + optional string bucketKey = 1; // e.g., /volume/bucket + optional uint64 bucketObjID = 2; // bucket's object ID, in case the bucket is deleted and recreated with same name + optional uint64 lifecycleConfigurationUpdateID = 3; // lifecycle configuration update ID, in case the bucket is updated with new rules + optional uint64 scanStartTime = 4; // Epoch time when this full scan started + optional uint64 scanEndTime = 5; // Epoch time when this full scan is completed + optional string lastScannedKey = 6; // the last scanned key in the bucket(for both OBS and FSO) + optional string lastScannedDir = 7; // the last scanned dir path, e.g /dir1/dir2/dir3 + optional string lastScannedDirKey = 8; // the last scanned dir key in directoryTable, e.g /0/1/3/dir3 + optional string lastScannedMpuKey = 9; +} +``` + +### OM DB Schema Updates +Add a new table `lifecycleStateTable` to `OMMetadataManager` to store the scan states: +- **Table Name:** `lifecycleStateTable` +- **Key:** `bucketKey` (String, e.g., `/volumeName/bucketName`) +- **Value:** `LifecycleScanState` + +### When to Persist the Pointer +Persisting the pointer for every key would overwhelm Ratis and RocksDB. We should checkpoint periodically: + +1. **Piggybacking on Deletes:** Add an optional `LifecycleScanState` field to `DeleteKeysRequest`. When the OM state machine applies the deletion, it atomically updates the `lifecycleStateTable` with the new pointer. This guarantees exactly-once semantics for the scan pointer relative to deletions. +2. **Move to trash**: Since there is no `RenameKeysRequest`, rename has be called multiple times for a batch of keys. We introduce a new OM request `SaveLifecycleScanStateRequest`. After a batch of keys are moved to trash, call `SaveLifecycleScanStateRequest` explicitly to persist the state. +3. **Periodic Standalone Checkpoints:** If no keys are expired (e.g., scanning millions of valid keys), we still need to save progress. The `LifecycleActionTask` will send this request periodically (e.g., every 100,000 keys iterated, or every 1 minute of execution time). +3. **End of Scan:** When the scan for a bucket finishes, a `SaveLifecycleScanStateRequest` is sent to mark state as completed by recording the completion time. + +### How to Resume the Scan +When `KeyLifecycleService` schedules a `LifecycleActionTask` for a bucket, it first reads the `LifecycleScanState` from the `lifecycleStateTable`. + +- **OBS/Legacy Resumption:** + The iterator for `keyTable` is initialized to seek to `lastScannedKey` instead of the bucket prefix. + ```java + TableIterator> keyTblItr = keyTable.iterator(bucketPrefix); + if (state.getLastScannedKey() != null) { + keyTblItr.seek(state.getLastScannedKey()); + // skip the exact match since it was already processed + } + ``` + +- **FSO Resumption:** + Since FSO bucket is iterated via a Depth-First Search (DFS) way, any directory that is after the `lastScannedDir` in the traversal path can be skipped. + +- **MPU Resumption:** + // TODO: implement MPU resumption + The `multipartInfoTable` iterator seeks to `lastScannedMpuKey` and continues. diff --git a/hadoop-hdds/docs/content/feature/Lifecycle.md b/hadoop-hdds/docs/content/feature/Lifecycle.md index 709bdf51f543..ec2302918bc6 100644 --- a/hadoop-hdds/docs/content/feature/Lifecycle.md +++ b/hadoop-hdds/docs/content/feature/Lifecycle.md @@ -132,19 +132,22 @@ For FILE_SYSTEM_OPTIMIZED (FSO) buckets, the Prefix must be a normalized and val - Cannot start with `/`. FSO bucket prefixes are relative to the bucket root and do not need a leading slash. - Cannot contain consecutive slashes `//`. - Path components cannot contain `.` (current directory), `..` (parent directory), or `:`. +- Must end with "/", or "" for root. The following table shows examples of valid and invalid prefixes: | Prefix | Valid for FSO Bucket | Reason | -|--------|----------------------|--------| +|----|----------------------|--| | `logs/` | Valid | Normalized directory prefix | | `data/2024/` | Valid | Multi-level directory prefix | -| `archive` | Valid | Simple prefix without slash | +| `archive` | Invalid | Without tailing slash | | `/logs/` | Invalid | Cannot start with `/`, use `logs/` instead | | `data//backup/` | Invalid | Contains consecutive slashes `//`, use `data/backup/` instead | | `data/../secret/` | Invalid | Contains `..`, parent directory references are not allowed | | `data/./logs/` | Invalid | Contains `.`, current directory references are not allowed | | `.Trash/` | Invalid | Cannot point to trash directories | +| `` | Valid | Point to Bucket's root directory | +| `/` | Invalid | It doesn't point to Bucket's root directory. Use "" instead | ## S3 Gateway API @@ -354,9 +357,9 @@ ozone admin om lifecycle resume [-id=] [-host=] In an OM HA deployment, the lifecycle service only runs on the leader OM. When a Transfer Leader operation is performed: 1. The lifecycle evaluation tasks running on the old leader will be interrupted. -2. After the new leader is elected, the lifecycle service restarts from the beginning. Previously evaluated buckets are not skipped, and the task starts over from the first bucket. +2. After the new leader is elected, the lifecycle service will skip the previously evaluated buckets/bucket contents, starting from the interrupted bucket. -Therefore, in scenarios with frequent leader transfers, it is recommended to monitor the actual execution of the lifecycle service to ensure expired objects are cleaned up in a timely manner. +In scenarios with frequent leader transfers, it is recommended to monitor the actual execution of the lifecycle service to ensure expired objects are cleaned up in a timely manner. ### Impact of Mass Key Expiration on Metadata Performance diff --git a/hadoop-hdds/docs/content/feature/Lifecycle.zh.md b/hadoop-hdds/docs/content/feature/Lifecycle.zh.md index b8aaa0f9d3eb..cab07a278647 100644 --- a/hadoop-hdds/docs/content/feature/Lifecycle.zh.md +++ b/hadoop-hdds/docs/content/feature/Lifecycle.zh.md @@ -131,19 +131,22 @@ FSO Bucket 的额外校验规则： - 不能以 `/` 开头。FSO Bucket 的 Prefix 是相对于 Bucket 根目录的路径，无需前导斜杠。 - 不能包含连续的斜杠 `//`。 - 路径组件中不能包含 `.`（当前目录）、`..`（父目录）或 `:`。 +- 路径必须以 `/` 结尾， 或者 “” 代表root。 以下是合法与不合法 Prefix 的对照示例： | Prefix | FSO Bucket 是否合法 | 原因 | -|--------|---------------------|------| +|----|--------------------|-----| | `logs/` | 合法 | 规范化的目录前缀 | | `data/2024/` | 合法 | 多级目录前缀 | -| `archive` | 合法 | 无斜杠的简单前缀 | +| `archive` | 不合法 | 无斜杠结尾 | | `/logs/` | 不合法 | 不能以 `/` 开头，应使用 `logs/` | | `data//backup/` | 不合法 | 包含连续斜杠 `//`，应使用 `data/backup/` | | `data/../secret/` | 不合法 | 包含 `..`，不允许使用父目录引用 | | `data/./logs/` | 不合法 | 包含 `.`，不允许使用当前目录引用 | | `.Trash/` | 不合法 | 不能指向回收站目录 | +| `` | 合法 | 指向Bucket 根目录 | +| `/` | 不合法 | 它不指向 Bucket 根目录。正确代表根目录的前缀是 “” | ## S3 Gateway API @@ -353,9 +356,9 @@ ozone admin om lifecycle resume [-id=] [-host=] 在 OM HA 部署中，生命周期服务仅在 Leader OM 上运行。当执行 Transfer Leader 操作时： 1. 旧 Leader 上正在运行的生命周期评估任务会被中断。 -2. 新 Leader 当选后，会重新从头启动生命周期服务，已经评估过的 Bucket 不会被跳过，任务将从第一个 Bucket 重新开始。 +2. 新 Leader 当选后，会重新从头启动生命周期服务，已经评估过的 Bucket 会被跳过，任务将从中断的Bucket 重新开始。 -因此，在频繁进行 Leader 切换的场景下，建议关注生命周期服务的实际执行情况，确保过期对象能被及时清理。 +在频繁进行 Leader 切换的场景下，建议关注生命周期服务的实际执行情况，确保过期对象能被及时清理。 ### 大批量 Key 过期对元数据性能的影响 diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 67ea6380fdf4..575b8d2aec5a 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@ -327,6 +327,7 @@ public static boolean isReadOnly(OMRequest omRequest) { case SetLifecycleConfiguration: case DeleteLifecycleConfiguration: case SetLifecycleServiceStatus: + case SaveLifecycleScanState: case UnknownCommand: return false; case EchoRPC: @@ -462,6 +463,7 @@ public static boolean shouldSendToFollower(OMRequest omRequest) { case SetLifecycleConfiguration: case DeleteLifecycleConfiguration: case SetLifecycleServiceStatus: + case SaveLifecycleScanState: case UnknownCommand: return false; case EchoRPC: diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 7857c28f3397..709f1f4d94e0 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -199,6 +199,14 @@ public final class OMConfigKeys { "ozone.lifecycle.service.delete.cached.directory.max-count"; public static final long OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT = 1000000; + // Save task state for every 5m, or evaluated keys reaches 100k + public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS = + "ozone.lifecycle.service.state.save.interval.ms"; + public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT = 5 * 60 * 1000; + public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED = + "ozone.lifecycle.service.state.save.keys.processed"; + public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT = 100000; + public static final String OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED = "ozone.lifecycle.service.move.to.trash.enabled"; public static final boolean diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java index 10b627790a03..ce9f56c30b89 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.helpers; +import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX; import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix; import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength; import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix; @@ -28,7 +29,6 @@ import net.jcip.annotations.Immutable; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule; @@ -61,7 +61,7 @@ private OmLCRule() { private OmLCRule(Builder builder) { this.prefix = builder.prefix; if (this.prefix != null) { - this.directoryStylePrefix = this.prefix.contains(OzoneConsts.OM_KEY_PREFIX); + this.directoryStylePrefix = this.prefix.contains(OM_KEY_PREFIX); } else { this.directoryStylePrefix = false; } @@ -196,6 +196,14 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti OMException.ResultCodes.INVALID_REQUEST); } action.valid(creationTime); + + if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + if (getEffectivePrefix() != null && !getEffectivePrefix().isEmpty() && + !getEffectivePrefix().endsWith(OM_KEY_PREFIX)) { + throw new OMException("FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'.", + OMException.ResultCodes.INVALID_REQUEST); + } + } } if (prefix != null && filter != null) { @@ -323,7 +331,6 @@ public static OmLCRule getFromProtobuf(LifecycleRule lifecycleRule, BucketLayout if (lifecycleRule.hasFilter()) { builder.setFilter(OmLCFilter.getFromProtobuf(lifecycleRule.getFilter(), layout)); } - return builder.build(); } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java new file mode 100644 index 000000000000..41bc5de16aa0 --- /dev/null +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import org.apache.hadoop.hdds.utils.db.Codec; +import org.apache.hadoop.hdds.utils.db.DelegatedCodec; +import org.apache.hadoop.hdds.utils.db.Proto2Codec; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState; + +/** + * POJO for LifecycleScanState. + */ +public class OmLifecycleScanState { + private String bucketKey; + private long bucketObjID; + private long lifecycleConfigurationUpdateID; + private long scanStartTime; + private Long scanEndTime; + private String lastScannedKey; + private String lastScannedDir; + private String lastScannedDirKey; + + private static final Codec CODEC = new DelegatedCodec<>( + Proto2Codec.get(LifecycleScanState.getDefaultInstance()), + OmLifecycleScanState::getFromProtobuf, + OmLifecycleScanState::getProtobuf, + OmLifecycleScanState.class); + + public static Codec getCodec() { + return CODEC; + } + + public OmLifecycleScanState(String bucketKey, long scanStartTime) { + this.bucketKey = bucketKey; + this.scanStartTime = scanStartTime; + } + + private OmLifecycleScanState(Builder builder) { + this.bucketKey = builder.bucketKey; + this.bucketObjID = builder.bucketObjID; + this.lifecycleConfigurationUpdateID = builder.lifecycleConfigurationUpdateID; + this.scanStartTime = builder.scanStartTime; + this.scanEndTime = builder.scanEndTime; + this.lastScannedKey = builder.lastScannedKey; + this.lastScannedDir = builder.lastScannedDir; + this.lastScannedDirKey = builder.lastScannedDirKey; + } + + public String getBucketKey() { + return bucketKey; + } + + public long getBucketObjID() { + return bucketObjID; + } + + public long getLifecycleConfigurationUpdateID() { + return lifecycleConfigurationUpdateID; + } + + public long getScanStartTime() { + return scanStartTime; + } + + public Long getScanEndTime() { + return scanEndTime; + } + + public void setScanEndTime(Long scanEndTime) { + this.scanEndTime = scanEndTime; + } + + public String getLastScannedKey() { + return lastScannedKey; + } + + public void setLastScannedKey(String lastScannedKey) { + this.lastScannedKey = lastScannedKey; + } + + public String getLastScannedDir() { + return lastScannedDir; + } + + public void setLastScannedDir(String dir) { + this.lastScannedDir = dir; + } + + public String getLastScannedDirKey() { + return lastScannedDirKey; + } + + public LifecycleScanState getProtobuf() { + LifecycleScanState.Builder builder = LifecycleScanState.newBuilder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketObjID) + .setLifecycleConfigurationUpdateID(lifecycleConfigurationUpdateID) + .setScanStartTime(scanStartTime); + + if (scanEndTime != null) { + builder.setScanEndTime(scanEndTime); + } + if (lastScannedKey != null) { + builder.setLastScannedKey(lastScannedKey); + } + if (lastScannedDir != null) { + builder.setLastScannedDir(lastScannedDir); + } + if (lastScannedDirKey != null) { + builder.setLastScannedDirKey(lastScannedDirKey); + } + return builder.build(); + } + + public static OmLifecycleScanState getFromProtobuf(LifecycleScanState proto) { + Builder builder = new Builder() + .setBucketKey(proto.getBucketKey()) + .setBucketObjID(proto.getBucketObjID()) + .setLifecycleConfigurationUpdateID(proto.getLifecycleConfigurationUpdateID()) + .setScanStartTime(proto.getScanStartTime()); + + if (proto.hasScanEndTime()) { + builder.setScanEndTime(proto.getScanEndTime()); + } + if (proto.hasLastScannedKey()) { + builder.setLastScannedKey(proto.getLastScannedKey()); + } + if (proto.hasLastScannedDir()) { + builder.setLastScannedDir(proto.getLastScannedDir()); + } + if (proto.hasLastScannedDirKey()) { + builder.setLastScannedDirKey(proto.getLastScannedDirKey()); + } + return builder.build(); + } + + @Override + public String toString() { + return "OmLifecycleScanState{" + + "bucketKey='" + bucketKey + '\'' + + ", bucketObjID=" + bucketObjID + + ", lifecycleConfigurationUpdateID=" + lifecycleConfigurationUpdateID + + ", scanStartTime=" + scanStartTime + + ", scanEndTime=" + scanEndTime + + ", lastScannedKey='" + lastScannedKey + '\'' + + ", lastScannedDir='" + lastScannedDir + '\'' + + ", lastScannedDirKey='" + lastScannedDirKey + '\'' + + '}'; + } + + public Builder toBuilder() { + return new Builder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketObjID) + .setLifecycleConfigurationUpdateID(lifecycleConfigurationUpdateID) + .setScanStartTime(scanStartTime) + .setScanEndTime(scanEndTime) + .setLastScannedKey(lastScannedKey) + .setLastScannedDir(lastScannedDir) + .setLastScannedDirKey(lastScannedDirKey); + } + + /** + * Builder for OmLifecycleScanState. + */ + public static class Builder { + private String bucketKey; + private long bucketObjID; + private long lifecycleConfigurationUpdateID; + private long scanStartTime; + private Long scanEndTime; + private String lastScannedKey; + private String lastScannedDir; + private String lastScannedDirKey; + + public Builder setBucketKey(String bucketKey) { + this.bucketKey = bucketKey; + return this; + } + + public long getBucketObjID() { + return bucketObjID; + } + + public Builder setBucketObjID(long bucketObjID) { + this.bucketObjID = bucketObjID; + return this; + } + + public long getLifecycleConfigurationUpdateID() { + return lifecycleConfigurationUpdateID; + } + + public Builder setLifecycleConfigurationUpdateID(long lifecycleConfigurationUpdateID) { + this.lifecycleConfigurationUpdateID = lifecycleConfigurationUpdateID; + return this; + } + + public Builder setScanStartTime(long scanStartTime) { + this.scanStartTime = scanStartTime; + return this; + } + + public Builder setScanEndTime(Long scanEndTime) { + this.scanEndTime = scanEndTime; + return this; + } + + public Builder setLastScannedKey(String keyTableKey) { + this.lastScannedKey = keyTableKey; + return this; + } + + public String getLastScannedKey() { + return lastScannedKey; + } + + public String getLastScannedDir() { + return lastScannedDir; + } + + public Builder setLastScannedDir(String dirTableKey) { + this.lastScannedDir = dirTableKey; + return this; + } + + public String getLastScannedDirKey() { + return lastScannedDirKey; + } + + public Builder setLastScannedDirKey(String lastScannedDirKey) { + this.lastScannedDirKey = lastScannedDirKey; + return this; + } + + public OmLifecycleScanState build() { + return new OmLifecycleScanState(this); + } + } +} diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java index 15e72e5a8c37..8b39e6b7aac2 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java @@ -104,6 +104,39 @@ public void testCreateInValidOmLCRule() throws OMException { "Filter and Prefix cannot both be null."); } + @Test + public void testCreateFSOLCRule() throws OMException { + long currentTime = System.currentTimeMillis(); + OmLCExpiration exp = new OmLCExpiration.Builder() + .setDays(30) + .build(); + + OmLCRule.Builder r1 = new OmLCRule.Builder() + .setId("remove Spark logs after 30 days") + .setEnabled(true) + .setPrefix("spark/logs") + .setAction(exp); + assertOMException(() -> r1.build().valid(BucketLayout.FILE_SYSTEM_OPTIMIZED, currentTime), + INVALID_REQUEST, "FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'"); + + OmLCRule.Builder r2 = new OmLCRule.Builder() + .setEnabled(true) + .setPrefix("spark/logs/") + .setAction(exp); + assertDoesNotThrow(() -> r2.build().valid(BucketLayout.FILE_SYSTEM_OPTIMIZED, currentTime)); + + OmLCRule.Builder r3 = new OmLCRule.Builder() + .setEnabled(true) + .setPrefix("") + .setAction(exp); + OmLCRule omLCRule = assertDoesNotThrow(r3::build); + assertDoesNotThrow(() -> omLCRule.valid(BucketLayout.FILE_SYSTEM_OPTIMIZED, currentTime)); + + // Empty id should generate a 48 (default) bit one. + assertEquals(OmLCRule.LC_ID_LENGTH, omLCRule.getId().length(), + "Expected a " + OmLCRule.LC_ID_LENGTH + " length generated ID"); + } + @Test public void testMultipleActionsInRule() throws OMException { long currentTime = System.currentTimeMillis(); diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java new file mode 100644 index 000000000000..59244bb06ce0 --- /dev/null +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.helpers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState; +import org.junit.jupiter.api.Test; + +/** + * Tests for OmLifecycleScanState. + */ +public class TestOmLifecycleScanState { + + @Test + public void testBuilderAndProtobufConversion() { + OmLifecycleScanState state = new OmLifecycleScanState.Builder() + .setBucketKey("/vol1/bucket1") + .setScanStartTime(123456789L) + .setScanEndTime(123456799L) + .setLastScannedKey("key1") + .setLastScannedDir("subDir1") + .build(); + + assertEquals("/vol1/bucket1", state.getBucketKey()); + assertEquals(123456789L, state.getScanStartTime()); + assertEquals(123456799L, state.getScanEndTime()); + assertEquals("key1", state.getLastScannedKey()); + assertEquals("subDir1", state.getLastScannedDir()); + + LifecycleScanState proto = state.getProtobuf(); + OmLifecycleScanState decodedState = OmLifecycleScanState.getFromProtobuf(proto); + + assertEquals(state.getBucketKey(), decodedState.getBucketKey()); + assertEquals(state.getScanStartTime(), decodedState.getScanStartTime()); + assertEquals(state.getScanEndTime(), decodedState.getScanEndTime()); + assertEquals(state.getLastScannedKey(), decodedState.getLastScannedKey()); + assertEquals(state.getLastScannedDir(), decodedState.getLastScannedDir()); + } + + @Test + public void testBuilderAndProtobufConversionWithoutOptionals() { + OmLifecycleScanState state = new OmLifecycleScanState.Builder() + .setBucketKey("/vol1/bucket1") + .setScanStartTime(123456789L) + .build(); + + assertEquals("/vol1/bucket1", state.getBucketKey()); + assertEquals(123456789L, state.getScanStartTime()); + assertNull(state.getScanEndTime()); + assertNull(state.getLastScannedKey()); + assertNull(state.getLastScannedDir()); + + LifecycleScanState proto = state.getProtobuf(); + OmLifecycleScanState decodedState = OmLifecycleScanState.getFromProtobuf(proto); + + assertEquals(state.getBucketKey(), decodedState.getBucketKey()); + assertEquals(state.getScanStartTime(), decodedState.getScanStartTime()); + assertNull(decodedState.getScanEndTime()); + assertNull(decodedState.getLastScannedKey()); + assertNull(decodedState.getLastScannedDir()); + } +} diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index d0a110b7263f..f2b21a06ad3c 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -162,6 +162,7 @@ enum Type { DeleteLifecycleConfiguration = 146; GetLifecycleServiceStatus = 147; SetLifecycleServiceStatus = 148; + SaveLifecycleScanState = 149; } enum SafeMode { @@ -319,6 +320,7 @@ message OMRequest { optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 147; optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 148; optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 149; + optional SaveLifecycleScanStateRequest saveLifecycleScanStateRequest = 150; } message OMResponse { @@ -459,6 +461,7 @@ message OMResponse { optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 146; optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 147; optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 148; + optional SaveLifecycleScanStateResponse saveLifecycleScanStateResponse = 149; } enum Status { @@ -1414,6 +1417,7 @@ message DeleteKeyRequest { message DeleteKeysRequest { optional DeleteKeyArgs deleteKeys = 1; optional RequestSource sourceType = 2 [default = USER]; + optional LifecycleScanState scanState = 3; } enum RequestSource { @@ -2624,3 +2628,22 @@ message SetLifecycleServiceStatusRequest { message SetLifecycleServiceStatusResponse { } + +message LifecycleScanState { + optional string bucketKey = 1; // volume/bucket + optional uint64 bucketObjID = 2; + optional uint64 lifecycleConfigurationUpdateID = 3; + optional uint64 scanStartTime = 4; + optional uint64 scanEndTime = 5; + optional string lastScannedKey = 6; + optional string lastScannedDir = 7; + optional string lastScannedDirKey = 8; + optional string lastScannedMpuKey = 9; +} + +message SaveLifecycleScanStateRequest { + optional LifecycleScanState state = 1; +} + +message SaveLifecycleScanStateResponse { +} diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index a664379426dc..88e4a03b2cdb 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -49,6 +49,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; @@ -499,6 +500,13 @@ String getMultipartKeyFSO(String volume, String bucket, String key, String Table getLifecycleConfigurationTable(); + /** + * Gets the LifecycleScanStateTable. + * + * @return Table + */ + Table getLifecycleScanStateTable(); + /** * @return list all LifecycleConfigurations. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index faa91bb67cf0..29a7a3943faa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -109,6 +109,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; @@ -178,6 +179,7 @@ public class OmMetadataManagerImpl implements OMMetadataManager, private Table transactionInfoTable; private Table metaTable; private Table lifecycleConfigurationTable; + private Table lifecycleScanStateTable; // Tables required for multi-tenancy private Table tenantAccessIdTable; @@ -512,6 +514,7 @@ protected void initializeOmTables(CacheType cacheType, compactionLogTable = initializer.get(OMDBDefinition.COMPACTION_LOG_TABLE_DEF); lifecycleConfigurationTable = initializer.get(OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE_DEF, cacheType); + lifecycleScanStateTable = initializer.get(OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE_DEF, cacheType); } /** @@ -1719,6 +1722,11 @@ public Table getLifecycleConfigurationTable() return lifecycleConfigurationTable; } + @Override + public Table getLifecycleScanStateTable() { + return lifecycleScanStateTable; + } + /** * @return list all LifecycleConfigurations. */ diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java index ba583258e83e..79284fa0a84f 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/codec/OMDBDefinition.java @@ -35,6 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartPartKey; @@ -334,6 +335,13 @@ public final class OMDBDefinition extends DBDefinition.WithMap { StringCodec.get(), OmLifecycleConfiguration.getCodec()); + public static final String LIFECYCLE_SCAN_STATE_TABLE = + "lifecycleScanStateTable"; + public static final DBColumnFamilyDefinition LIFECYCLE_SCAN_STATE_TABLE_DEF + = new DBColumnFamilyDefinition<>(LIFECYCLE_SCAN_STATE_TABLE, + StringCodec.get(), + OmLifecycleScanState.getCodec()); + //--------------------------------------------------------------------------- private static final Map> COLUMN_FAMILIES = DBColumnFamilyDefinition.newUnmodifiableMap( @@ -360,7 +368,8 @@ public final class OMDBDefinition extends DBDefinition.WithMap { TRANSACTION_INFO_TABLE_DEF, USER_TABLE_DEF, VOLUME_TABLE_DEF, - LIFECYCLE_CONFIGURATION_TABLE_DEF); + LIFECYCLE_CONFIGURATION_TABLE_DEF, + LIFECYCLE_SCAN_STATE_TABLE_DEF); private static final OMDBDefinition INSTANCE = new OMDBDefinition(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index dcab37b348d7..04dc20b0afcd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -65,6 +65,7 @@ import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationDeleteRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleConfigurationSetRequest; +import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleSaveScanStateRequest; import org.apache.hadoop.ozone.om.request.lifecycle.OMLifecycleSetServiceStatusRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3ExpiredMultipartUploadsAbortRequest; import org.apache.hadoop.ozone.om.request.s3.security.OMSetSecretRequest; @@ -351,6 +352,8 @@ public static OMClientRequest createClientRequest(OMRequest omRequest, return new OMLifecycleConfigurationDeleteRequest(omRequest); case SetLifecycleServiceStatus: return new OMLifecycleSetServiceStatusRequest(omRequest); + case SaveLifecycleScanState: + return new OMLifecycleSaveScanStateRequest(omRequest); default: throw new OMException("Unrecognized write command type request " + cmdType, OMException.ResultCodes.INVALID_REQUEST); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index 7ee4400f52cc..e1d9a66ce3fd 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; @@ -75,6 +76,7 @@ import org.apache.hadoop.ozone.request.validation.RequestProcessingPhase; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -162,6 +164,17 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut validateBucketAndVolume(omMetadataManager, volumeName, bucketName); String volumeOwner = getVolumeOwner(omMetadataManager, volumeName); + if (sourceType == RequestSource.LIFECYCLE && deleteKeyRequest.hasScanState()) { + if (ozoneManager.getAclsEnabled()) { + UserGroupInformation ugi = createUGIForApi(); + if (!ozoneManager.isAdmin(ugi)) { + throw new OMException("Access denied for user " + ugi + ". " + + "Superuser privilege is required to save Lifecycle Service task state.", + OMException.ResultCodes.ACCESS_DENIED); + } + } + } + for (indexFailed = 0; indexFailed < length; indexFailed++) { String keyName = deleteKeyArgs.getKeys(indexFailed); String objectKey = @@ -227,10 +240,19 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut quotaReleasedEmptyKeys.getValue(), true); omBucketInfo.decrUsedNamespace(quotaReleasedEmptyKeys.getValue(), false); + OmLifecycleScanState state = null; + if (sourceType == RequestSource.LIFECYCLE && deleteKeyRequest.hasScanState()) { + state = OmLifecycleScanState.getFromProtobuf(deleteKeyRequest.getScanState()); + // Update cache + ozoneManager.getMetadataManager().getLifecycleScanStateTable() + .addCacheEntry(new CacheKey<>(state.getBucketKey()), + CacheValue.get(trxnLogIndex, state)); + } + final long volumeId = omMetadataManager.getVolumeId(volumeName); omClientResponse = getOmClientResponse(ozoneManager, omKeyInfoList, dirList, omResponse, - unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap); + unDeletedKeys, keyToError, deleteStatus, omBucketInfo, volumeId, openKeyInfoMap, state); result = Result.SUCCESS; long endNanosDeleteKeySuccessLatencyNs = Time.monotonicNowNanos(); @@ -335,7 +357,8 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, Map keyToErrors, - boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { + boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap, + OmLifecycleScanState scanState) { OMClientResponse omClientResponse; List deleteKeyErrors = new ArrayList<>(); for (Map.Entry key : keyToErrors.entrySet()) { @@ -348,7 +371,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, .setUnDeletedKeys(unDeletedKeys).addAllErrors(deleteKeyErrors)) .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus) .build(), omKeyInfoList, - omBucketInfo.copyObject(), openKeyInfoMap); + omBucketInfo.copyObject(), openKeyInfoMap, scanState); return omClientResponse; } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java index a501739d0c31..51de2dd0753e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OmKeysDeleteRequestWithFSO.java @@ -37,6 +37,7 @@ import org.apache.hadoop.ozone.om.helpers.ErrorInfo; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.OMClientResponse; @@ -165,7 +166,8 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, OzoneManagerProtocolProtos.OMResponse.Builder omResponse, OzoneManagerProtocolProtos.DeleteKeyArgs.Builder unDeletedKeys, Map keyToErrors, - boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap) { + boolean deleteStatus, OmBucketInfo omBucketInfo, long volumeId, Map openKeyInfoMap, OmLifecycleScanState state) { OMClientResponse omClientResponse; List deleteKeyErrors = new ArrayList<>(); for (Map.Entry key : keyToErrors.entrySet()) { @@ -179,7 +181,7 @@ protected OMClientResponse getOmClientResponse(OzoneManager ozoneManager, .setStatus(deleteStatus).setUnDeletedKeys(unDeletedKeys).addAllErrors(deleteKeyErrors)) .setStatus(deleteStatus ? OK : PARTIAL_DELETE).setSuccess(deleteStatus) .build(), omKeyInfoList, dirList, - omBucketInfo.copyObject(), volumeId, openKeyInfoMap); + omBucketInfo.copyObject(), volumeId, openKeyInfoMap, state); return omClientResponse; } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java new file mode 100644 index 000000000000..94c4d3faa1b9 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.lifecycle; + +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.lifecycle.OMLifecycleSaveScanStateResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * Handles SaveLifecycleScanState request. + */ +public class OMLifecycleSaveScanStateRequest extends OMClientRequest { + + public OMLifecycleSaveScanStateRequest(OMRequest omRequest) { + super(omRequest); + } + + @Override + public OMRequest preExecute(OzoneManager ozoneManager) throws OMException { + if (ozoneManager.getAclsEnabled()) { + UserGroupInformation ugi = createUGIForApi(); + if (!ozoneManager.isAdmin(ugi)) { + throw new OMException("Access denied for user " + ugi + ". " + + "Superuser privilege is required to save Lifecycle Service task state.", + OMException.ResultCodes.ACCESS_DENIED); + } + } + return getOmRequest(); + } + + @Override + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { + SaveLifecycleScanStateRequest request = getOmRequest().getSaveLifecycleScanStateRequest(); + OmLifecycleScanState state = OmLifecycleScanState.getFromProtobuf(request.getState()); + + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + + // Update cache + ozoneManager.getMetadataManager().getLifecycleScanStateTable() + .addCacheEntry(new CacheKey<>(state.getBucketKey()), + CacheValue.get(context.getIndex(), state)); + + return new OMLifecycleSaveScanStateResponse(omResponse.build(), state); + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java index 3cb1220b83ce..a8eff37bf661 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java @@ -20,6 +20,7 @@ import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.BUCKET_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_KEY_TABLE; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.PARTIAL_DELETE; @@ -35,26 +36,29 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; /** * Response for DeleteKey request. */ -@CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE}) +@CleanupTableInfo(cleanupTables = {KEY_TABLE, OPEN_KEY_TABLE, DELETED_TABLE, BUCKET_TABLE, LIFECYCLE_SCAN_STATE_TABLE}) public class OMKeysDeleteResponse extends AbstractOMKeyDeleteResponse { private List omKeyInfoList; private OmBucketInfo omBucketInfo; private Map openKeyInfoMap = new HashMap<>(); + private OmLifecycleScanState scanState; public OMKeysDeleteResponse(@Nonnull OMResponse omResponse, @Nonnull List keyDeleteList, @Nonnull OmBucketInfo omBucketInfo, - @Nonnull Map openKeyInfoMap) { + @Nonnull Map openKeyInfoMap, OmLifecycleScanState scanState) { super(omResponse); this.omKeyInfoList = keyDeleteList; this.omBucketInfo = omBucketInfo; this.openKeyInfoMap = openKeyInfoMap; + this.scanState = scanState; } /** @@ -107,6 +111,11 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, batchOperation, entry.getKey(), entry.getValue()); } } + + if (scanState != null) { + omMetadataManager.getLifecycleScanStateTable().putWithBatch( + batchOperation, scanState.getBucketKey(), scanState); + } } public List getOmKeyInfoList() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java index 0b283509354e..e955f45a980e 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DELETED_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.DIRECTORY_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.OPEN_FILE_TABLE; import jakarta.annotation.Nonnull; @@ -34,6 +35,7 @@ import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@ -41,21 +43,24 @@ * Response for DeleteKeys request. */ @CleanupTableInfo(cleanupTables = { FILE_TABLE, OPEN_FILE_TABLE, DIRECTORY_TABLE, - DELETED_DIR_TABLE, DELETED_TABLE, BUCKET_TABLE }) + DELETED_DIR_TABLE, DELETED_TABLE, BUCKET_TABLE, LIFECYCLE_SCAN_STATE_TABLE}) public class OMKeysDeleteResponseWithFSO extends OMKeysDeleteResponse { private List dirsList; private long volumeId; + private OmLifecycleScanState scanState; public OMKeysDeleteResponseWithFSO( @Nonnull OzoneManagerProtocolProtos.OMResponse omResponse, @Nonnull List keyDeleteList, @Nonnull List dirDeleteList, @Nonnull OmBucketInfo omBucketInfo, @Nonnull long volId, - @Nonnull Map openKeyInfoMap) { - super(omResponse, keyDeleteList, omBucketInfo, openKeyInfoMap); + @Nonnull Map openKeyInfoMap, + OmLifecycleScanState scanState) { + super(omResponse, keyDeleteList, omBucketInfo, openKeyInfoMap, scanState); this.dirsList = dirDeleteList; this.volumeId = volId; + this.scanState = scanState; } @Override @@ -101,6 +106,11 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, batchOperation, entry.getKey(), entry.getValue()); } } + + if (scanState != null) { + omMetadataManager.getLifecycleScanStateTable().putWithBatch( + batchOperation, scanState.getBucketKey(), scanState); + } } @Override diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java new file mode 100644 index 000000000000..0f94293269b7 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle/OMLifecycleSaveScanStateResponse.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.lifecycle; + +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE; +import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK; + +import java.io.IOException; +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; +import org.apache.hadoop.ozone.om.response.CleanupTableInfo; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + +/** + * Response for SaveLifecycleScanState request. + */ +@CleanupTableInfo(cleanupTables = {LIFECYCLE_SCAN_STATE_TABLE}) +public class OMLifecycleSaveScanStateResponse extends OMClientResponse { + + private OmLifecycleScanState state; + + public OMLifecycleSaveScanStateResponse(OMResponse omResponse, OmLifecycleScanState state) { + super(omResponse); + this.state = state; + } + + @Override + public void addToDBBatch(OMMetadataManager omMetadataManager, BatchOperation batchOperation) throws IOException { + if (getOMResponse().getStatus() == OK) { + omMetadataManager.getLifecycleScanStateTable().putWithBatch( + batchOperation, state.getBucketKey(), state); + } + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index dd37430dccfa..a14bc0ed0a05 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -30,6 +30,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_MPU_ABORT_LIMIT_PER_TASK_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.OBJECT_STORE; import com.google.common.annotations.VisibleForTesting; @@ -42,7 +46,8 @@ import java.time.Instant; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.Deque; import java.util.HashSet; import java.util.Iterator; @@ -83,6 +88,7 @@ import org.apache.hadoop.ozone.om.helpers.OmLCFilter; import org.apache.hadoop.ozone.om.helpers.OmLCRule; import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartUpload; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -99,6 +105,7 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RenameKeyRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RequestSource; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.ratis.protocol.ClientId; @@ -125,10 +132,14 @@ public class KeyLifecycleService extends BackgroundService { private final ConcurrentHashMap inFlight; private OMMetadataManager omMetadataManager; private int ratisByteLimit; + private long stateSaveIntervalMs; + private long maxKeysProcessedPerState; private ClientId clientId = ClientId.randomId(); private AtomicLong callId = new AtomicLong(0); private OzoneTrash ozoneTrash; private static List injectors; + private static boolean test = false; + private static List consolidatedRuleList; public KeyLifecycleService(OzoneManager ozoneManager, KeyManager manager, long serviceInterval, @@ -154,6 +165,22 @@ public KeyLifecycleService(OzoneManager ozoneManager, OZONE_KEY_LIFECYCLE_SERVICE_ENABLED_DEFAULT)); this.moveToTrashEnabled = new AtomicBoolean(conf.getBoolean(OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED, OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED_DEFAULT)); + this.stateSaveIntervalMs = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT); + if (!test && stateSaveIntervalMs <= 0) { + LOG.warn("Illegal value {} for Property {}. Set {} to {}", stateSaveIntervalMs, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT); + maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; + } + this.maxKeysProcessedPerState = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT); + if (!test && maxKeysProcessedPerState <= 0) { + LOG.warn("Illegal value {} for Property {}. Set {} to {}", maxKeysProcessedPerState, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT); + maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT; + } this.inFlight = new ConcurrentHashMap(); this.omMetadataManager = ozoneManager.getMetadataManager(); int limit = (int) conf.getStorageSize( @@ -278,6 +305,20 @@ public final class LifecycleActionTask implements BackgroundTask { private long numDirRenamed = 0; private long numMultipartUploadIterated = 0; private long numMultipartUploadAborted = 0; + private String lastScannedKey; + private String lastScannedDir; + private String lastScannedDirKey; + + private long lastStateSaveTime = Time.monotonicNow(); + private long lastStateSaveKeyCount = 0; + + private boolean shouldSaveState() { + if ((Time.monotonicNow() - lastStateSaveTime) > stateSaveIntervalMs || + (numKeyIterated - lastStateSaveKeyCount) >= maxKeysProcessedPerState) { + return true; + } + return false; + } public LifecycleActionTask(OmLifecycleConfiguration lcConfig) { this.policy = lcConfig; @@ -296,6 +337,7 @@ public BackgroundTaskResult call() { if (shouldRun()) { LOG.info("Running LifecycleActionTask {}", bucketKey); taskStartTime = Time.monotonicNow(); + lastStateSaveTime = taskStartTime; OmBucketInfo bucket; try { if (getInjector(0) != null) { @@ -319,76 +361,108 @@ public BackgroundTaskResult call() { return result; } - List originRuleList = policy.getRules(); - // remove disabled rules - List ruleList = originRuleList.stream().filter(r -> r.isEnabled()).collect(Collectors.toList()); - - List expirationRules = ruleList.stream() - .filter(r -> r.getExpiration() != null) - .collect(Collectors.toList()); - List mpuRules = ruleList.stream() - .filter(r -> r.getAbortIncompleteMultipartUpload() != null) - .collect(Collectors.toList()); - - if (!expirationRules.isEmpty()) { - LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize); - LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize); - Table keyTable = omMetadataManager.getKeyTable(bucket.getBucketLayout()); - /** - * Filter treatment. - * "" - all objects - * "/" - if it's OBS/Legacy, means keys starting with "/"; If it's FSO, not supported - * "/key" - if it's OBS/Legacy, means keys starting with "/key", "/" is literally "/"; - * If it's FSO, means keys or dirs starting with "key", "/" will be treated as separator mark. - * "key" - if it's OBS/Legacy, means keys starting with "key"; - * if it's FSO, means keys for dirs starting with "key" too. - * "dir/" - if it's OBS/Legacy, means keys starting with "dir/"; - * - if it's FSO, means keys/dirs under directory "dir", doesn't include directory "dir" itself. - * - For FSO bucket, as directory ModificationTime will not be updated when any of its child key/subdir - * changes, so remember to add the tailing slash "/" when configure prefix, otherwise the whole - * directory will be expired and deleted once its ModificationTime meats the condition. - */ - if (bucket.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { - OmVolumeArgs volume; - try { - volume = omMetadataManager.getVolumeTable().get(omMetadataManager.getVolumeKey(bucket.getVolumeName())); - if (volume == null) { - LOG.warn("Volume {} cannot be found, might be deleted during this task's execution", - bucket.getVolumeName()); + OmLifecycleScanState.Builder scanStateBuilder = null; + try { + OmLifecycleScanState scanState = omMetadataManager.getLifecycleScanStateTable().get(bucketKey); + if (scanState == null || (scanState.getBucketObjID() != bucket.getObjectID() || + scanState.getLifecycleConfigurationUpdateID() != policy.getUpdateID() || + scanState.getScanEndTime() != null)) { + scanStateBuilder = new OmLifecycleScanState.Builder(); + scanStateBuilder.setBucketKey(bucketKey); + scanStateBuilder.setScanStartTime(System.currentTimeMillis()); + scanStateBuilder.setBucketObjID(bucket.getObjectID()); + scanStateBuilder.setLifecycleConfigurationUpdateID(policy.getUpdateID()); + LOG.info("Create/Recreate OmLifecycleScanState for {} bucket {} bucketID {} " + + "lifecycleConfigurationUpdateID {}", bucket.getBucketLayout(), bucketKey, bucket.getObjectID(), + policy.getUpdateID()); + } else { + scanStateBuilder = scanState.toBuilder(); + LOG.info("Resume OmLifecycleScanState {}", scanState); + } + } catch (Exception e) { + LOG.warn("Failed to get scan state for bucket {}", bucketKey, e); + } + + try { + List originRuleList = policy.getRules(); + // remove disabled rules + List ruleList = originRuleList.stream().filter( + r -> r.isEnabled()).collect(Collectors.toList()); + + List expirationRules = ruleList.stream() + .filter(r -> r.getExpiration() != null) + .collect(Collectors.toList()); + List mpuRules = ruleList.stream() + .filter(r -> r.getAbortIncompleteMultipartUpload() != null) + .collect(Collectors.toList()); + + if (!expirationRules.isEmpty()) { + LimitedExpiredObjectList expiredKeyList = new LimitedExpiredObjectList(listMaxSize); + LimitedExpiredObjectList expiredDirList = new LimitedExpiredObjectList(listMaxSize); + Table keyTable = omMetadataManager.getKeyTable(bucket.getBucketLayout()); + /** + * Filter treatment. + * "" - all objects + * "/" - if it's OBS/Legacy, means keys starting with "/"; If it's FSO, not supported + * "/key" - if it's OBS/Legacy, means keys starting with "/key", "/" is literally "/"; + * If it's FSO, means keys or dirs starting with "key", "/" will be treated as separator mark. + * "key" - if it's OBS/Legacy, means keys starting with "key"; + * if it's FSO, means keys for dirs starting with "key" too. + * "dir/" - if it's OBS/Legacy, means keys starting with "dir/"; + * - if it's FSO, means keys/dirs under directory "dir", doesn't include directory "dir" itself. + * - For FSO bucket, as directory ModificationTime will not be updated when any of its child + * key/subdir changes, so remember to add the tailing slash "/" when configure prefix, otherwise + * the whole directory will be expired and deleted once its ModificationTime meats the condition. + */ + if (bucket.getBucketLayout() == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + OmVolumeArgs volume; + try { + volume = omMetadataManager.getVolumeTable().get(omMetadataManager.getVolumeKey(bucket.getVolumeName())); + if (volume == null) { + LOG.warn("Volume {} cannot be found, might be deleted during this task's execution", + bucket.getVolumeName()); + onFailure(bucketKey); + return result; + } + } catch (IOException e) { + LOG.warn("Failed to get volume {}", bucket.getVolumeName(), e); onFailure(bucketKey); return result; } - } catch (IOException e) { - LOG.warn("Failed to get volume {}", bucket.getVolumeName(), e); - onFailure(bucketKey); - return result; + evaluateFSOBucket(volume, bucket, bucketKey, keyTable, expirationRules, expiredKeyList, + expiredDirList, scanStateBuilder); + } else { + // use bucket name as key iterator prefix + evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList, scanStateBuilder); } - evaluateFSOBucket(volume, bucket, bucketKey, keyTable, expirationRules, expiredKeyList, expiredDirList); - } else { - // use bucket name as key iterator prefix - evaluateBucket(bucket, keyTable, expirationRules, expiredKeyList); - } - if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) { - LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey); - } else { - LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}", - expiredKeyList.size(), expiredDirList.size(), bucketKey); - - // If trash is enabled, move files to trash, instead of send delete requests. - // OBS bucket doesn't support trash. - if (bucket.getBucketLayout() == OBJECT_STORE) { - sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList, false); + if (expiredKeyList.isEmpty() && expiredDirList.isEmpty()) { + LOG.info("No expired keys/dirs found/remained for bucket {}", bucketKey); + sendSaveScanStateRequest(scanStateBuilder, true); } else { - // handle keys first, then directories - handleAndClearFullList(bucket, expiredKeyList, false); - handleAndClearFullList(bucket, expiredDirList, true); + LOG.info("{} expired keys and {} expired dirs found and remained for bucket {}", + expiredKeyList.size(), expiredDirList.size(), bucketKey); + + // If trash is enabled, move files to trash, instead of send delete requests. + // OBS bucket doesn't support trash. + if (bucket.getBucketLayout() == OBJECT_STORE) { + sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList, + false, scanStateBuilder, true); + } else { + // handle keys first, then directories + handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, true); + handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, true); + } } } - } - if (!mpuRules.isEmpty()) { - processMultipartUploads(bucket, mpuRules); + if (!mpuRules.isEmpty()) { + processMultipartUploads(bucket, mpuRules); + } + } catch (Throwable e) { + LOG.error("Failed to evaluate lifecycle configuration for bucket {}", bucketKey, e); + onFailure(bucketKey); + return result; } onSuccess(bucketKey); @@ -401,108 +475,226 @@ public BackgroundTaskResult call() { @SuppressWarnings("checkstyle:parameternumber") private void evaluateFSOBucket(OmVolumeArgs volume, OmBucketInfo bucket, String bucketKey, Table keyTable, List ruleList, - LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList) { + LimitedExpiredObjectList expiredKeyList, LimitedExpiredObjectList expiredDirList, + OmLifecycleScanState.Builder scanStateBuilder) { List prefixRuleList = ruleList.stream().filter(r -> r.isPrefixEnable()).collect(Collectors.toList()); // r.isPrefixEnable() == false means empty filter List noPrefixRuleList = ruleList.stream().filter(r -> !r.isPrefixEnable()).collect(Collectors.toList()); - for (OmLCRule rule : prefixRuleList) { - // find KeyInfo of each directory for prefix - List dirList; + if (!noPrefixRuleList.isEmpty()) { + // evaluate all rules against each key + prefixRuleList.addAll(noPrefixRuleList); + evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "", + prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder); + return; + } + + List unionPrefixRuleList = + getRuleUnion(volume.getObjectID(), bucket, prefixRuleList, bucketKey); + + if (unionPrefixRuleList != null) { + if (unionPrefixRuleList.isEmpty()) { + // fallback to evaluate the whole bucket + evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, "", + prefixRuleList, expiredKeyList, expiredDirList, scanStateBuilder); + } else { + for (RuleListWithDirectoryList ruleWithDirList : unionPrefixRuleList) { + List rules = ruleWithDirList.getRuleList(); + DirectoryList dir = ruleWithDirList.getDirList(); + evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, dir.getLastSubDirPath(), + dir.getLastSubDir(), dir.getLastSubDirKey(), + rules, expiredKeyList, expiredDirList, scanStateBuilder); + } + } + } + } + + /** + * Finds the directory union list from a list of prefixes and sorts them + * according to the FSO depth-first iteration order. + */ + private List getRuleUnion(long volumeId, OmBucketInfo bucket, + List rules, String bucketKey) { + + if (rules.isEmpty() || rules.stream().anyMatch( + r -> r.getEffectivePrefix() == null || r.getEffectivePrefix().isEmpty())) { + // The union of anything with the root is just the root itself. + return new ArrayList(); + } + + List effectiveRuleList = new ArrayList<>(); + for (OmLCRule rule : rules) { + String prefix = rule.getEffectivePrefix(); + // Resolve each prefix to actual FSO directories in the DB try { - dirList = getDirList(volume, bucket, rule.getEffectivePrefix(), bucketKey); + if (!prefix.endsWith(OzoneConsts.OM_KEY_PREFIX)) { + // FSO bucket doesn't allow prefix without tailing '/' + // Prefix ends with a slash, it explicitly refers to a directory (e.g. "log/") + LOG.warn("Skip rule {} since FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'", rule); + continue; + } + + // Normalize by removing the trailing slash for uniform comparison + String normalizedPrefix = prefix.substring(0, prefix.length() - 1); + DirectoryList dirList = getDirList(volumeId, bucket, normalizedPrefix, bucketKey); + // If the prefix is log/, and "log" dir really exists, then the matched dir is "log". + // Otherwise, this rule doesn't match any dir/file in this FSO bucket, this rule can be skipped. + if (!dirList.isEmpty() && dirList.isAllResolvedPrefix()) { + RuleListWithDirectoryList ruleListWithDirectoryList = new RuleListWithDirectoryList( + Collections.singletonList(rule), dirList, prefix); + effectiveRuleList.add(ruleListWithDirectoryList); + } } catch (IOException e) { - LOG.warn("Skip rule {} as its prefix doesn't have all directory exist", rule); - // skip this rule if some directory doesn't exist for this rule's prefix + // Directory doesn't exist or IO error, skip this rule + LOG.warn("Skip to evaluate rule {} due to failed to resolve prefix {} for bucket {}", + rule, prefix, bucketKey, e); + } + } + + if (effectiveRuleList.isEmpty()) { + // there is no valid rule found, either prefix doesn't end with "/", + // or any directory along the prefix cannot be found. + LOG.warn("Prefix of all rules of bucket {} cannot be resolved to an existing directory. ", bucketKey); + return null; + } + + if (effectiveRuleList.size() == 1) { + return effectiveRuleList; + } + + // Find if one rule's prefix is the sub string of another rule's prefix. + // e.g. + // dir1/dir2/, dir1/dir2/dir3/, dir1/ -> dir1/ + // dir1/dir2/, dir1/dir3/, dir1/dir4/ -> dir1/dir2/, dir1/dir3/, dir1/dir4dir1/ + // dir1/dir2/dir3/, dir1/dir2/, dir2/ -> dir1/dir2/, dir2/ + // dir1/dir2/, dir1/dir3/, dir1/ -> dir1/ + List consolidatedRules = new ArrayList<>(); + Set skipEvaluatedRuleList = new HashSet<>(); + for (int i = 0; i < effectiveRuleList.size(); i++) { + OmLCRule rule = effectiveRuleList.get(i).getRuleList().get(0); + if (skipEvaluatedRuleList.contains(rule)) { continue; } - StringBuffer lastDirPath = new StringBuffer(); - OmDirectoryInfo lastDir = null; - if (!dirList.isEmpty()) { - lastDir = dirList.get(dirList.size() - 1); - for (int i = 0; i < dirList.size(); i++) { - lastDirPath.append(dirList.get(i).getName()); - if (i != dirList.size() - 1) { - lastDirPath.append(OM_KEY_PREFIX); - } - } - if (lastDirPath.toString().startsWith(TRASH_PREFIX)) { - LOG.info("Skip evaluate trash directory {}", lastDirPath); - } else { - evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, lastDirPath.toString(), lastDir, - Arrays.asList(rule), expiredKeyList, expiredDirList); + + RuleListWithDirectoryList consolidatedCandidate = new RuleListWithDirectoryList(); + String consolidatedPrefix = effectiveRuleList.get(i).getConsolidatedPrefix(); + String finalRuleIndexID = rule.getId(); + DirectoryList finalDirList = effectiveRuleList.get(i).getDirList(); + for (int j = i + 1; j < effectiveRuleList.size(); j++) { + OmLCRule otherRule = effectiveRuleList.get(j).getRuleList().get(0); + if (skipEvaluatedRuleList.contains(otherRule)) { + continue; } - if (!rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) { - // if the prefix doesn't end with "/", then also search and evaluate the directory itself - // for example, "dir1/dir2" matches both directory "dir1/dir2" and "dir1/dir22" - // or "dir1" matches both directory "dir1" and "dir11" - long objID; - String objPrefix; - String objPath; - if (dirList.size() > 1) { - OmDirectoryInfo secondLastDir = dirList.get(dirList.size() - 2); - objID = secondLastDir.getObjectID(); - objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX + bucket.getObjectID() + - OM_KEY_PREFIX + secondLastDir.getObjectID(); - StringBuffer secondLastDirPath = new StringBuffer(); - for (int i = 0; i < dirList.size() - 1; i++) { - secondLastDirPath.append(dirList.get(i).getName()); - if (i != dirList.size() - 2) { - secondLastDirPath.append(OM_KEY_PREFIX); - } - } - objPath = secondLastDirPath.toString(); - } else { - objID = bucket.getObjectID(); - objPrefix = OM_KEY_PREFIX + volume.getObjectID() + OM_KEY_PREFIX + bucket.getObjectID() + - OM_KEY_PREFIX + bucket.getObjectID(); - objPath = ""; - } - try { - SubDirectorySummary subDirSummary = getSubDirectory(objID, objPrefix, omMetadataManager); - for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) { - String subDirPath = objPath.isEmpty() ? subDir.getName() : objPath + OM_KEY_PREFIX + subDir.getName(); - if (!subDir.getName().equals(TRASH_PREFIX) && subDirPath.startsWith(rule.getEffectivePrefix()) && - (lastDir == null || subDir.getObjectID() != lastDir.getObjectID())) { - evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, subDirPath, subDir, - Arrays.asList(rule), expiredKeyList, expiredDirList); - } - } - } catch (IOException e) { - // log failure and continue the process - LOG.warn("Failed to get sub directories of {} under {}/{}", objPrefix, - bucket.getVolumeName(), bucket.getBucketName(), e); - return; - } + DirectoryList otherDirList = effectiveRuleList.get(j).getDirList(); + String otherPrefix = otherRule.getEffectivePrefix(); + if (otherPrefix.startsWith(consolidatedPrefix)) { + LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. " + + " Consolidate {} into {}.", otherRule.getId(), otherPrefix, finalRuleIndexID, + consolidatedPrefix, otherRule.getId(), finalRuleIndexID); + consolidatedCandidate.addRule(otherRule); + skipEvaluatedRuleList.add(otherRule); + } else if (consolidatedPrefix.startsWith(otherPrefix)) { + LOG.info("Rule {}'s prefix {} is sub string of rule {}'s prefix {}. Consolidate {} int {}. ", + consolidatedPrefix, consolidatedPrefix, otherRule.getId(), otherPrefix, consolidatedPrefix, + otherRule.getId()); + consolidatedPrefix = otherPrefix; + finalRuleIndexID = otherRule.getId(); + finalDirList = otherDirList; + consolidatedCandidate.addRule(otherRule); + skipEvaluatedRuleList.add(otherRule); } - } else { - evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, - Arrays.asList(rule), expiredKeyList, expiredDirList); } + + consolidatedCandidate.addRule(rule); + consolidatedCandidate.setDirList(finalDirList); + consolidatedCandidate.setConsolidatedPrefix(consolidatedPrefix); + consolidatedRules.add(consolidatedCandidate); } - if (!noPrefixRuleList.isEmpty()) { - evaluateKeyAndDirTable(bucket, volume.getObjectID(), keyTable, "", null, - noPrefixRuleList, expiredKeyList, expiredDirList); + // Sort the list of paths lexicographically. + // FSO Depth-First Search order evaluates directories in lexicographical order + // (since it retrieves entries from RocksDB sorted by name within the same parent). + // Standard string sort on logical paths separated by "/" perfectly matches this DFS order. + List sortedConsolidatedRules = + consolidatedRules.stream().sorted(new RuleListWithDirectoryListOrder()).collect(Collectors.toList()); + + LOG.info("Final consolidated rules: " + + sortedConsolidatedRules.stream().map(RuleListWithDirectoryList::toString).collect(Collectors.joining(", "))); + if (test) { + consolidatedRuleList = sortedConsolidatedRules; } + return sortedConsolidatedRules; + } + + private boolean canSkipDir(OmDirectoryInfo currentDir, String currentDirTableKey, DirectoryList dirList) { + // currentDir null is bucket root + if (currentDir == null) { + return false; + } + + int count = dirList.getSubDirCount(); + // if currentDir is equal to lastScannedDir + if (currentDir.getObjectID() == dirList.getSubDirList().get(count - 1).getObjectID()) { + return false; + } + + // if currentDir is parent of lastScannedDir + long currentObjID = currentDir.getObjectID(); + for (int i = 0; i < count; i++) { + OmDirectoryInfo dir = dirList.getSubDirList().get(i); + if (dir.getObjectID() == currentObjID) { + return false; + } + } + + // if currentDir and lastScannedDir has same parent + do { + long parentID = currentDir.getParentObjectID(); + for (int i = 0; i < count; i++) { + OmDirectoryInfo dir = dirList.getSubDirList().get(i); + if (dir.getParentObjectID() == parentID) { + if (dirList.getSubDirKeyList().get(i).compareTo(currentDirTableKey) < 0) { + return true; + } + } + } + return false; + } while (true); } @SuppressWarnings({"checkstyle:parameternumber", "checkstyle:MethodLength"}) private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table keyTable, - String directoryPath, @Nullable OmDirectoryInfo dir, List ruleList, LimitedExpiredObjectList keyList, - LimitedExpiredObjectList dirList) { + String directoryPath, @Nullable OmDirectoryInfo dir, String dirKey, List ruleList, + LimitedExpiredObjectList keyList, LimitedExpiredObjectList dirList, + OmLifecycleScanState.Builder scanStateBuilder) { String volumeName = bucket.getVolumeName(); String bucketName = bucket.getBucketName(); LimitedSizeStack stack = new LimitedSizeStack(cachedDirMaxCount); + String lastScannedDirInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDir(); + String lastScannedDirKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDirKey(); + String lastScannedKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedKey(); + DirectoryList lastScannedDirList = null; + String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); + if (lastScannedDirInState != null && lastScannedDirKeyInState != null) { + // find all parents of lastScannedSubDir + try { + lastScannedDirList = getDirList(volumeObjId, bucket, lastScannedDirInState, bucketKey); + } catch (IOException e) { + // Saved lastScannedDir in state could be deleted or renamed after it's saved. + // Fallback to no state saved status. + LOG.info("Failed to get DirList for lastScannedDirInState {}", lastScannedDirInState, e); + lastScannedDirInState = null; + } + } try { if (dir != null) { - stack.push(new PendingEvaluateDirectory(dir, directoryPath, null)); + stack.push(new PendingEvaluateDirectory(dir, dirKey, directoryPath, null)); } else { // put a placeholder PendingEvaluateDirectory to stack for bucket - stack.push(new PendingEvaluateDirectory(null, "", null)); + stack.push(new PendingEvaluateDirectory(null, "", "", null)); } } catch (CapacityFullException e) { LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, directoryPath != null ? directoryPath : "", e); @@ -518,10 +710,55 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table getOzoneManager() != null ? getOzoneManager().isLeaderReady() : "N/A"); return; } + PendingEvaluateDirectory item = stack.pop(); OmDirectoryInfo currentDir = item.getDirectoryInfo(); String currentDirPath = item.getDirPath(); long currentDirObjID = currentDir == null ? bucket.getObjectID() : currentDir.getObjectID(); + String currentDirTableKey = item.getDirTableKey(); + + /** + * / + * dir1 dir2 dir3 dir30 + * / \ / \ + * dir4 dir5 dir6 dir7 + * / \ + * dir8 dir9 + * lastScannedDir = dir3/dir6/dir8, which means + * Scanned: + * dir30 + * dir3/dir7 + * dir3/dir6/dir9 + * Half scanned: + * dir3/dir6/dir8 + * Not scanned: + * dir3/dir6 + * dir3 + * dir1/dir5 + * dir/dir4 + * dir1 + * directoryTable table key format : /volumeId/bucketId/parentId/dirName + * based on the depth first evaluation order, and stack push posh iteration pattern + * - dir1, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned + * - dir2, on grand level of lastScannedDir, and name order < lastScannedDir grand, not scanned + * - dir3, grand of lastScannedDir, not scanned + * - dir30, on grand level of lastScannedDir, and name order > lastScannedDir grand, scanned, skip + * - dir3/dir6, parent of lastScannedDir, not scanned + * - dir3/dir7, parent level of lastScannedDir, and name order > lastScannedDir parent, scanned, skip + * - dir3/dir8, lastScannedDir, partially scanned, + * - dir3/dir9, same the same parentID as lastScannedDir, and name order > lastScannedDir, scanned, skip + */ + if (lastScannedDirList != null && + canSkipDir(currentDir, currentDirTableKey, lastScannedDirList)) { + LOG.info("Skip {} in LifecycleActionTask for bucket {}. ", currentDirPath, bucketName); + continue; + } + + lastScannedDir = currentDirPath; + lastScannedDirKey = currentDirTableKey; + if (shouldSaveState()) { + flushAndSaveState(bucket, keyList, dirList, scanStateBuilder); + } // use current directory's object ID to iterate the keys and directories under it String prefix = @@ -529,10 +766,10 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table LOG.debug("Prefix {} for {}/{}", prefix, bucket.getVolumeName(), bucket.getBucketName()); // get direct sub directories - SubDirectorySummary subDirSummary = item.getSubDirSummary(); + DirectoryList subDirSummary; boolean newSubDirPushed = false; long deletedDirCount = 0; - if (subDirSummary == null) { + if (item.isFirstEvaluate()) { try { subDirSummary = getSubDirectory(currentDirObjID, prefix, omMetadataManager); } catch (IOException e) { @@ -540,7 +777,17 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table LOG.warn("Failed to get sub directories of {} under {}/{}", currentDirPath, volumeName, bucketName, e); continue; } + } else { + // this item is a parent directory, check how many sub directories are deleted. + subDirSummary = item.getSubDirSummary(); + for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) { + if (deletedDirSet.remove(subDir.getObjectID())) { + deletedDirCount++; + } + } + } + if (item.isFirstEvaluate()) { // filter sub directory list if (!subDirSummary.getSubDirList().isEmpty()) { Iterator iterator = subDirSummary.getSubDirList().iterator(); @@ -550,6 +797,7 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table currentDirPath + OM_KEY_PREFIX + subDir.getName(); if (subDirPath.startsWith(TRASH_PREFIX)) { iterator.remove(); + continue; } boolean matched = false; for (OmLCRule rule : ruleList) { @@ -565,7 +813,8 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table } if (!subDirSummary.getSubDirList().isEmpty()) { - item.setSubDirSummary(subDirSummary); + item.setDirectoryList(subDirSummary); + item.setFirstEvaluate(false); try { stack.push(item); } catch (CapacityFullException e) { @@ -574,11 +823,13 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table } // depth first evaluation, push subDirs into stack - for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) { + for (int i = 0; i < subDirSummary.getSubDirCount(); i++) { + OmDirectoryInfo subDir = subDirSummary.getSubDirList().get(i); String subDirPath = currentDirPath.isEmpty() ? subDir.getName() : currentDirPath + OM_KEY_PREFIX + subDir.getName(); try { - stack.push(new PendingEvaluateDirectory(subDir, subDirPath, null)); + stack.push(new PendingEvaluateDirectory(subDir, subDirSummary.getSubDirKeyList().get(i), + subDirPath, null)); } catch (CapacityFullException e) { LOG.warn("Abort evaluate {}/{} at {}", volumeName, bucketName, subDirPath, e); return; @@ -586,13 +837,6 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table } newSubDirPushed = true; } - } else { - // this item is a parent directory, check how many sub directories are deleted. - for (OmDirectoryInfo subDir : subDirSummary.getSubDirList()) { - if (deletedDirSet.remove(subDir.getObjectID())) { - deletedDirCount++; - } - } } if (newSubDirPushed) { @@ -628,39 +872,75 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table // mark key as expired, check next key if (keyList.isFull()) { // if keyList is full, send delete/rename request for expired keys - handleAndClearFullList(bucket, keyList, false); + handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false); } keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID()); numKeysExpired++; break; } } + lastScannedKey = entry.getKey().getCacheKey(); } } try (TableIterator> keyTblItr = keyTable.iterator(prefix)) { + if (lastScannedDirKeyInState != null && lastScannedDirKeyInState.compareTo(currentDirTableKey) == 0 && + lastScannedKeyInState != null && lastScannedKeyInState.startsWith(prefix)) { + LOG.info("Seek to key {} under directory {}", scanStateBuilder.getLastScannedKey(), lastScannedDirInState); + keyTblItr.seek(scanStateBuilder.getLastScannedKey()); + if (keyTblItr.hasNext()) { + Table.KeyValue first = keyTblItr.next(); + if (!first.getKey().equals(scanStateBuilder.getLastScannedKey())) { + OmKeyInfo key = first.getValue(); + String keyPath = currentDirPath.isEmpty() ? key.getKeyName() : + currentDirPath + OM_KEY_PREFIX + key.getKeyName(); + if (!deletedKeySetInCache.remove(first.getKey()) && !keySetInCache.remove(first.getKey())) { + numKeyIterated++; + numKeysUnderDir++; + for (OmLCRule rule : ruleList) { + if (key.getParentObjectID() == currentDirObjID && rule.match(key, keyPath)) { + if (keyList.isFull()) { + handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false); + } + keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID()); + numKeysExpired++; + break; + } + } + lastScannedKey = first.getKey(); + } + } + } + } + while (keyTblItr.hasNext()) { + if (shouldSaveState()) { + LOG.info("Saving scan state for bucket {} at key {}", bucketName, lastScannedKey); + flushAndSaveState(bucket, keyList, dirList, scanStateBuilder); + } Table.KeyValue keyValue = keyTblItr.next(); OmKeyInfo key = keyValue.getValue(); String keyPath = currentDirPath.isEmpty() ? key.getKeyName() : currentDirPath + OM_KEY_PREFIX + key.getKeyName(); - numKeyIterated++; if (deletedKeySetInCache.remove(keyValue.getKey()) || keySetInCache.remove(keyValue.getKey())) { continue; } + numKeyIterated++; numKeysUnderDir++; for (OmLCRule rule : ruleList) { if (key.getParentObjectID() == currentDirObjID && rule.match(key, keyPath)) { // mark key as expired, check next key if (keyList.isFull()) { // if keyList is full, send delete request for pending deletion keys - handleAndClearFullList(bucket, keyList, false); + handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false); } keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID()); numKeysExpired++; + break; } } + lastScannedKey = keyValue.getKey(); } } catch (IOException e) { // log failure and continue the process other directories in stack @@ -671,25 +951,49 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table // if this directory is empty or all files/subDirs are expired, evaluate itself if ((numKeysUnderDir == 0 && subDirSummary.getSubDirCount() == 0) || (numKeysUnderDir == numKeysExpired && deletedDirCount == subDirSummary.getSubDirCount())) { - for (OmLCRule rule : ruleList) { - String path = (rule.getEffectivePrefix() != null && rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX)) ? + List pathList = new ArrayList<>(); + boolean skipDir = false; + for (int i = 0; i < ruleList.size(); i++) { // NOPMD + OmLCRule rule = ruleList.get(i); + String path = rule.getEffectivePrefix() != null && rule.getEffectivePrefix().endsWith(OM_KEY_PREFIX) ? currentDirPath + OM_KEY_PREFIX : currentDirPath; + if (path != null && path.equals(rule.getEffectivePrefix())) { + LOG.info("Prefix directory {} doesn't get expired", path); + skipDir = true; + break; + } + pathList.add(path); + } + + if (skipDir) { + continue; + } + for (int i = 0; i < ruleList.size(); i++) { + String path = pathList.get(i); + OmLCRule rule = ruleList.get(i); if (currentDir != null && rule.match(currentDir, path)) { if (dirList.isFull()) { - // if expiredDirList is full, send delete request for pending deletion directories - handleAndClearFullList(bucket, dirList, true); + // if expiredDirList is full, send delete request for both pending deletion keys and directories + handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false); + handleAndClearFullList(bucket, dirList, true, scanStateBuilder, false); + if (getInjector(2) != null && getInjector(2).getException() != null) { + Throwable ex = getInjector(2).getException(); + getInjector(2).setException(null); + throw new RuntimeException(ex); + } } dirList.add(currentDirPath, 0, currentDir.getUpdateID()); deletedDirSet.add(currentDir.getObjectID()); + break; } } } } } - private SubDirectorySummary getSubDirectory(long dirObjID, String prefix, OMMetadataManager metaMgr) + private DirectoryList getSubDirectory(long dirObjID, String prefix, OMMetadataManager metaMgr) throws IOException { - SubDirectorySummary subDirList = new SubDirectorySummary(); + DirectoryList subDirList = new DirectoryList(); // Check all dirTable cache for any sub paths. Table dirTable = metaMgr.getDirectoryTable(); @@ -706,7 +1010,7 @@ private SubDirectorySummary getSubDirectory(long dirObjID, String prefix, OMMeta continue; } if (cacheOmDirInfo.getParentObjectID() == dirObjID) { - subDirList.addSubDir(cacheOmDirInfo); + subDirList.addSubDir(entry.getKey().getCacheKey(), cacheOmDirInfo, cacheOmDirInfo.getName()); } } @@ -721,46 +1025,75 @@ private SubDirectorySummary getSubDirectory(long dirObjID, String prefix, OMMeta continue; } if (dir.getParentObjectID() == dirObjID) { - subDirList.addSubDir(dir); + subDirList.addSubDir(entry.getKey(), dir, dir.getName()); } } } return subDirList; } + private void flushAndSaveState(OmBucketInfo bucket, LimitedExpiredObjectList expiredKeyList, + LimitedExpiredObjectList expiredDirList, OmLifecycleScanState.Builder scanStateBuilder) { + boolean saved = false; + if (expiredKeyList != null && !expiredKeyList.isEmpty()) { + if (bucket.getBucketLayout() == OBJECT_STORE) { + sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), expiredKeyList, + false, scanStateBuilder, false); + } else { + handleAndClearFullList(bucket, expiredKeyList, false, scanStateBuilder, false); + } + saved = true; + } + if (expiredDirList != null && !expiredDirList.isEmpty()) { + if (bucket.getBucketLayout() != OBJECT_STORE) { + handleAndClearFullList(bucket, expiredDirList, true, scanStateBuilder, false); + saved = true; + } + } + if (!saved) { + sendSaveScanStateRequest(scanStateBuilder, false); + } + lastStateSaveTime = Time.monotonicNow(); + lastStateSaveKeyCount = numKeyIterated; + } + private void evaluateBucket(OmBucketInfo bucketInfo, - Table keyTable, List ruleList, LimitedExpiredObjectList expiredKeyList) { + Table keyTable, List ruleList, LimitedExpiredObjectList expiredKeyList, + OmLifecycleScanState.Builder scanStateBuilder) { String volumeName = bucketInfo.getVolumeName(); String bucketName = bucketInfo.getBucketName(); + String bucketPrefix = omMetadataManager.getBucketKey(volumeName, bucketName); - // use bucket name as key iterator prefix try (TableIterator> keyTblItr = - keyTable.iterator(omMetadataManager.getBucketKey(volumeName, bucketName))) { + keyTable.iterator(bucketPrefix)) { + if (scanStateBuilder != null && scanStateBuilder.getLastScannedKey() != null) { + keyTblItr.seek(scanStateBuilder.getLastScannedKey()); + // Skip the exact match since it was already processed + if (keyTblItr.hasNext()) { + Table.KeyValue first = keyTblItr.next(); + if (!first.getKey().equals(scanStateBuilder.getLastScannedKey())) { + // We seeked past it, so we need to process this one. + // We can't easily "push back" in TableIterator, so we handle it here. + processKey(bucketInfo, first.getValue(), ruleList, expiredKeyList, scanStateBuilder); + numKeyIterated++; + lastScannedKey = first.getKey(); + } + } + } + while (keyTblItr.hasNext()) { if (!shouldRun()) { LOG.info("KeyLifecycleService is suspended or disabled. " + "Stopping LifecycleActionTask for bucket {}.", bucketName); return; } + if (shouldSaveState()) { + flushAndSaveState(bucketInfo, expiredKeyList, null, scanStateBuilder); + } Table.KeyValue keyValue = keyTblItr.next(); - OmKeyInfo key = keyValue.getValue(); + processKey(bucketInfo, keyValue.getValue(), ruleList, expiredKeyList, scanStateBuilder); numKeyIterated++; - if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY && - key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) { - LOG.info("Skip evaluate trash directory {} and all its child files and sub directories", TRASH_PREFIX); - continue; - } - for (OmLCRule rule : ruleList) { - if (rule.match(key)) { - // mark key as expired, check next key - if (expiredKeyList.isFull()) { - // if expiredKeyList is full, send delete/rename request for expired keys - handleAndClearFullList(bucketInfo, expiredKeyList, false); - } - expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(), key.getUpdateID()); - break; - } - } + lastScannedKey = keyValue.getKey(); } } catch (IOException e) { // log failure and continue the process to delete/move files already identified in this run @@ -768,6 +1101,30 @@ private void evaluateBucket(OmBucketInfo bucketInfo, } } + private void processKey(OmBucketInfo bucketInfo, OmKeyInfo key, List ruleList, + LimitedExpiredObjectList expiredKeyList, OmLifecycleScanState.Builder scanStateBuilder) { + if (bucketInfo.getBucketLayout() == BucketLayout.LEGACY && + key.getKeyName().startsWith(TRASH_PREFIX + OzoneConsts.OM_KEY_PREFIX)) { + return; + } + for (OmLCRule rule : ruleList) { + if (rule.match(key)) { + // mark key as expired, check next key + if (expiredKeyList.isFull()) { + // if expiredKeyList is full, send delete/rename request for expired keys + handleAndClearFullList(bucketInfo, expiredKeyList, false, scanStateBuilder, false); + if (getInjector(2) != null && getInjector(2).getException() != null) { + Throwable ex = getInjector(2).getException(); + getInjector(2).setException(null); + throw new RuntimeException(ex); + } + } + expiredKeyList.add(key.getKeyName(), key.getReplicatedSize(), key.getUpdateID()); + break; + } + } + } + /** * Process AbortIncompleteMultipartUpload actions for incomplete multipart uploads. * Iterates through the multipartInfoTable and aborts uploads that match the rule criteria @@ -792,7 +1149,6 @@ private void processMultipartUploads(OmBucketInfo bucketInfo, List rul "Stopping multipart upload processing for bucket {}.", bucketName); return; } - Table.KeyValue entry = mpuIterator.next(); OmMultipartKeyInfo mpuKeyInfo = entry.getValue(); numMultipartUploadIterated++; @@ -961,41 +1317,47 @@ private void abortExpiredMultipartUploadsAndClear(OmBucketInfo bucketInfo, } /** - * If prefix is /dir1/dir2, but dir1 doesn't exist, then it will return exception. - * If prefix is /dir1/dir2, but dir2 doesn't exist, then it will return a list with dir1 only. - * If prefix is /dir1/dir2, although dir1 exists, but get(dir1) failed with IOException, - * then it will return exception too. + * If the prefix is /dir1/dir2, but dir1 doesn't exist, then it will return an exception. + * If the prefix is /dir1/dir2, but dir2 doesn't exist, then it will return a list with dir1 only. + * If the prefix is /dir1/dir2, although dir1 exists, but get(dir1) failed with IOException, + * then it will return an exception too. */ - private List getDirList(OmVolumeArgs volume, OmBucketInfo bucket, String prefix, String bucketKey) + private DirectoryList getDirList(long volumeID, OmBucketInfo bucket, String prefix, String bucketKey) throws IOException { - // find KeyInfo of each directory for prefix + // find KeyInfo of each directory for the prefix java.nio.file.Path keyPath = Paths.get(prefix); Iterator elements = keyPath.iterator(); long lastKnownParentId = bucket.getObjectID(); - List dirList = new ArrayList<>(); + DirectoryList directoryList = new DirectoryList(); + StringBuffer currentDirPath = new StringBuffer(); while (elements.hasNext()) { String dirName = elements.next().toString(); String dbDirName = omMetadataManager.getOzonePathKey( - volume.getObjectID(), bucket.getObjectID(), - lastKnownParentId, dirName); + volumeID, bucket.getObjectID(), lastKnownParentId, dirName); try { OmDirectoryInfo omDirInfo = omMetadataManager.getDirectoryTable().get(dbDirName); - // It's OK there is no directory for the last part of prefix, which is probably not a directory + // It's OK there is no directory for the last part of the prefix, which is probably not a directory if (omDirInfo == null) { if (elements.hasNext()) { throw new OMException("Directory " + dbDirName + " does not exist for bucket " + bucketKey, OMException.ResultCodes.DIRECTORY_NOT_FOUND); } + directoryList.setNotAllResolvedPrefix(); } else { - dirList.add(omDirInfo); + if (currentDirPath.length() == 0) { + currentDirPath.append(dirName); + } else { + currentDirPath.append('/').append(dirName); + } + directoryList.addSubDir(dbDirName, omDirInfo, currentDirPath.toString()); lastKnownParentId = omDirInfo.getObjectID(); } } catch (IOException e) { - LOG.warn("Failed to get directory {} from bucket {}", dbDirName, bucketKey, e); - throw new IOException("Failed to get directory " + dbDirName + " from bucket " + bucketKey); + LOG.warn("Failed to get directory {} for bucket {}", dbDirName, bucketKey, e); + throw new IOException("Failed to get directory " + dbDirName + " for bucket " + bucketKey); } } - return dirList; + return directoryList; } private void onFailure(String bucketName) { @@ -1004,6 +1366,12 @@ private void onFailure(String bucketName) { metrics.incNumKeyIterated(numKeyIterated); metrics.incNumDirIterated(numDirIterated); metrics.incNumMultipartUploadIterated(numMultipartUploadIterated); + long timeSpent = Time.monotonicNow() - taskStartTime; + LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " + + "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " + + "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated, + numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted, + numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted); } private void onSuccess(String bucketName) { @@ -1015,23 +1383,62 @@ private void onSuccess(String bucketName) { metrics.incNumDirIterated(numDirIterated); metrics.incNumMultipartUploadIterated(numMultipartUploadIterated); metrics.incNumMultipartUploadAborted(numMultipartUploadAborted); - LOG.info("Spend {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " + + LOG.info("Spent {} ms on bucket {} to iterate {} keys and {} dirs and {} multipart uploads, " + "deleted {} keys with {} bytes, and {} dirs, renamed {} keys with {} bytes, and {} dirs to trash, " + "aborted {} multipart uploads", timeSpent, bucketName, numKeyIterated, numDirIterated, numMultipartUploadIterated, numKeyDeleted, sizeKeyDeleted, numDirDeleted, numKeyRenamed, sizeKeyRenamed, numDirRenamed, numMultipartUploadAborted); } - private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList, boolean dir) { + private void handleAndClearFullList(OmBucketInfo bucket, LimitedExpiredObjectList keysList, + boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) { if (moveToTrashEnabled.get() && bucket.getBucketLayout() != OBJECT_STORE && ozoneTrash != null) { moveToTrash(bucket, keysList, dir); + sendSaveScanStateRequest(scanStateBuilder, scanFinished); } else { - sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), keysList, dir); + sendDeleteKeysRequestAndClearList(bucket.getVolumeName(), bucket.getBucketName(), keysList, dir, + scanStateBuilder, scanFinished); } } - private void sendDeleteKeysRequestAndClearList(String volume, String bucket, - LimitedExpiredObjectList keysList, boolean dir) { + private void sendSaveScanStateRequest(OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) { + if (scanStateBuilder != null) { + if (lastScannedDir != null) { + scanStateBuilder.setLastScannedDir(lastScannedDir); + } + if (lastScannedDirKey != null) { + scanStateBuilder.setLastScannedDirKey(lastScannedDirKey); + } + if (lastScannedKey != null) { + scanStateBuilder.setLastScannedKey(lastScannedKey); + } + if (scanFinished) { + scanStateBuilder.setScanEndTime(System.currentTimeMillis()); + } + OmLifecycleScanState state = scanStateBuilder.build(); + + SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder() + .setState(state.getProtobuf()) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState) + .setVersion(ClientVersion.CURRENT_VERSION) + .setClientId(clientId.toString()) + .setSaveLifecycleScanStateRequest(saveRequest) + .build(); + + LOG.debug("Save scan state {}", state); + try { + OzoneManagerRatisUtils.submitRequest(getOzoneManager(), omRequest, clientId, callId.getAndIncrement()); + } catch (ServiceException e) { + LOG.error("Failed to submit SaveLifecycleScanState request", e); + } + } + } + + private void sendDeleteKeysRequestAndClearList(String volume, String bucket, LimitedExpiredObjectList keysList, + boolean dir, OmLifecycleScanState.Builder scanStateBuilder, boolean scanFinished) { try { if (getInjector(1) != null) { try { @@ -1053,10 +1460,29 @@ private void sendDeleteKeysRequestAndClearList(String volume, String bucket, builder.addAllUpdateIDs(keysList.updateIDSubList(startIndex, endIndex)); DeleteKeyArgs deleteKeyArgs = builder.build(); - DeleteKeysRequest deleteKeysRequest = DeleteKeysRequest.newBuilder() + DeleteKeysRequest.Builder requestBuilder = DeleteKeysRequest.newBuilder() .setDeleteKeys(deleteKeyArgs) - .setSourceType(RequestSource.LIFECYCLE) - .build(); + .setSourceType(RequestSource.LIFECYCLE); + + if (scanStateBuilder != null) { + if (lastScannedKey != null) { + scanStateBuilder.setLastScannedKey(lastScannedKey); + } + if (lastScannedDir != null) { + scanStateBuilder.setLastScannedDir(lastScannedDir); + } + if (lastScannedDirKey != null) { + scanStateBuilder.setLastScannedDirKey(lastScannedDirKey); + } + if (scanFinished) { + scanStateBuilder.setScanEndTime(System.currentTimeMillis()); + } + OmLifecycleScanState state = scanStateBuilder.build(); + requestBuilder.setScanState(state.getProtobuf()); + LOG.debug("Save scan state: {}", state); + } + + DeleteKeysRequest deleteKeysRequest = requestBuilder.build(); LOG.debug("request size {} for {} keys", deleteKeysRequest.getSerializedSize(), keyCount); if (deleteKeysRequest.getSerializedSize() < ratisByteLimit) { @@ -1323,6 +1749,12 @@ public void add(String name, long size, long updateID) { objectUpdateIDs.add(updateID); } + public void addAll(LimitedExpiredObjectList other) { + objectNames.addAll(other.objectNames); + objectReplicatedSize.addAll(other.objectReplicatedSize); + objectUpdateIDs.addAll(other.objectUpdateIDs); + } + public int size() { return objectNames.size(); } @@ -1395,6 +1827,10 @@ public void add(T element) { internalList.add(element); } + public void addAll(LimitedSizeList other) { + internalList.addAll(other.internalList); + } + public T get(int index) { return internalList.get(index); } @@ -1484,13 +1920,25 @@ public void clear() { */ public static class PendingEvaluateDirectory { private final OmDirectoryInfo directoryInfo; - private final String dirPath; - private SubDirectorySummary subDirSummary; + private String dirTableKey; + private String dirPath; + private DirectoryList directoryList; + private boolean firstEvaluate; - public PendingEvaluateDirectory(OmDirectoryInfo dir, String path, SubDirectorySummary summary) { + public PendingEvaluateDirectory(OmDirectoryInfo dir, String dirTableKey, String dirPath, DirectoryList summary) { this.directoryInfo = dir; - this.dirPath = path; - this.subDirSummary = summary; + this.dirTableKey = dirTableKey; + this.dirPath = dirPath; + this.directoryList = summary; + this.firstEvaluate = true; + } + + public String getDirTableKey() { + return dirTableKey; + } + + public void setDirTableKey(String tableKey) { + dirTableKey = tableKey; } public String getDirPath() { @@ -1501,28 +1949,41 @@ public OmDirectoryInfo getDirectoryInfo() { return directoryInfo; } - public SubDirectorySummary getSubDirSummary() { - return subDirSummary; + public DirectoryList getSubDirSummary() { + return directoryList; + } + + public void setDirectoryList(DirectoryList summary) { + directoryList = summary; } - public void setSubDirSummary(SubDirectorySummary summary) { - subDirSummary = summary; + public boolean isFirstEvaluate() { + return firstEvaluate; + } + + public void setFirstEvaluate(boolean firstEvaluate) { + this.firstEvaluate = firstEvaluate; } } /** - * An in-memory class to hold sub directory summary. + * An in-memory class to hold a directory list. */ - public static class SubDirectorySummary { + public static class DirectoryList { private final List subDirList; - private long subDirCount; + private final List subDirKeyList; + private final List subDirKeyPathList; + private int subDirCount; + private boolean allResolvedPrefix = true; - public SubDirectorySummary() { + public DirectoryList() { this.subDirList = new ArrayList<>(); + this.subDirKeyList = new ArrayList<>(); + this.subDirKeyPathList = new ArrayList<>(); this.subDirCount = 0; } - public long getSubDirCount() { + public int getSubDirCount() { return subDirCount; } @@ -1530,10 +1991,149 @@ public List getSubDirList() { return subDirList; } - public void addSubDir(OmDirectoryInfo dir) { + public List getSubDirKeyList() { + return subDirKeyList; + } + + public void addSubDir(String key, OmDirectoryInfo dir, String dirPath) { + subDirKeyList.add(key); subDirList.add(dir); + subDirKeyPathList.add(dirPath); subDirCount++; } + + public OmDirectoryInfo getLastSubDir() { + return subDirCount > 0 ? subDirList.get(subDirCount - 1) : null; + } + + public String getLastSubDirKey() { + return subDirCount > 0 ? subDirKeyList.get(subDirCount - 1) : null; + } + + public String getLastSubDirPath() { + return subDirCount > 0 ? subDirKeyPathList.get(subDirCount - 1) : null; + } + + public boolean isAllResolvedPrefix() { + return allResolvedPrefix; + } + + public void setNotAllResolvedPrefix() { + this.allResolvedPrefix = false; + } + + public boolean isEmpty() { + return subDirCount == 0; + } + + @Override + public String toString() { + return "DirectoryList { " + + "subDirList = " + subDirList + + ", subDirKeyList = " + subDirKeyList + + ", subDirKeyPathList = " + subDirKeyPathList + + ", subDirCount = " + subDirCount + + ", allResolvedPrefix = " + allResolvedPrefix + + '}'; + } + } + + /** + * An in-memory class to hold a rule list, together with the resolved prefix's directory list. + */ + public static class RuleListWithDirectoryList { + private DirectoryList dirList; + private final List ruleList; + private String consolidatedPrefix; + + public RuleListWithDirectoryList() { + this.ruleList = new ArrayList<>(); + this.dirList = new DirectoryList(); + } + + public RuleListWithDirectoryList(List ruleList, DirectoryList dirList, String prefix) { + this.ruleList = ruleList; + this.dirList = dirList; + this.consolidatedPrefix = prefix; + } + + public List getRuleList() { + return ruleList; + } + + public DirectoryList getDirList() { + return dirList; + } + + public void addRule(OmLCRule rule) { + this.ruleList.add(rule); + } + + public void setDirList(DirectoryList dirList) { + this.dirList = dirList; + } + + public String getConsolidatedPrefix() { + return consolidatedPrefix; + } + + public void setConsolidatedPrefix(String consolidatedPrefix) { + this.consolidatedPrefix = consolidatedPrefix; + } + + public boolean isEmpty() { + return dirList.isEmpty() && ruleList.isEmpty() && consolidatedPrefix == null; + } + + @Override + public String toString() { + return "RuleListWithDirectoryList { " + + "dirList = " + dirList + + ", ruleList = " + ruleList + + ", consolidatedPrefix = '" + consolidatedPrefix + '\'' + + '}'; + } + } + + /** + * Orders of RuleListWithDirectoryList. + */ + public static class RuleListWithDirectoryListOrder implements Comparator { + + public static final Comparator INSTANCE = + new RuleListWithDirectoryListOrder(); + + @Override + public int compare(RuleListWithDirectoryList o1, RuleListWithDirectoryList o2) { + List dirList1 = o1.getDirList().getSubDirList(); + List dirList2 = o2.getDirList().getSubDirList(); + + // find their shared parent directory + OmDirectoryInfo parent = null; + for (int i = dirList1.size() - 1; i >= 0; i--) { + long objID = dirList1.get(i).getObjectID(); + for (int j = dirList2.size() - 1; j >= 0; j--) { + if (dirList2.get(j).getObjectID() == objID) { + parent = dirList2.get(j); + break; + } + } + if (parent != null) { + break; + } + } + if (parent == null) { + // e.g, dir1 and dir2, dir2 should ahead of dir2, given the depth-first and dir's RocksDB key order + return dirList2.get(0).getName().compareTo(dirList1.get(0).getName()); + } else { + long parentID = parent.getObjectID(); + OmDirectoryInfo dir1 = dirList1.stream().filter(dir -> dir.getParentObjectID() == parentID) + .collect(Collectors.toList()).get(0); + OmDirectoryInfo dir2 = dirList2.stream().filter(dir -> dir.getParentObjectID() == parentID) + .collect(Collectors.toList()).get(0); + return dir2.getName().compareTo(dir1.getName()); + } + } } /** @@ -1572,4 +2172,16 @@ public CapacityFullException(String message) { super(message); } } + + public static void setTest(boolean test) { + KeyLifecycleService.test = test; + } + + public static void reSetConsolidatedRuleList() { + consolidatedRuleList = null; + } + + public static List getConsolidatedRuleList() { + return consolidatedRuleList; + } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java index 41be905235bb..799bd0ab3db1 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java @@ -33,6 +33,7 @@ import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.KEY_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE; +import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_INFO_TABLE; import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.MULTIPART_PARTS_TABLE; @@ -140,7 +141,8 @@ public class TestOmMetadataManager { SNAPSHOT_INFO_TABLE, SNAPSHOT_RENAMED_TABLE, COMPACTION_LOG_TABLE, - LIFECYCLE_CONFIGURATION_TABLE + LIFECYCLE_CONFIGURATION_TABLE, + LIFECYCLE_SCAN_STATE_TABLE }; private OMMetadataManager omMetadataManager; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java new file mode 100644 index 000000000000..96c0508a04b3 --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.lifecycle; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.UUID; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SaveLifecycleScanStateRequest; +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.jupiter.api.Test; + +/** + * Tests OMLifecycleSaveScanStateRequest. + */ +public class TestOMLifecycleSaveScanStateRequest { + + @Test + public void testPreExecuteAdminCheck() throws Exception { + OzoneManager ozoneManager = mock(OzoneManager.class); + + // Test when ACLs are enabled but user is not admin + when(ozoneManager.getAclsEnabled()).thenReturn(true); + when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState) + .setClientId(UUID.randomUUID().toString()) + .setSaveLifecycleScanStateRequest(SaveLifecycleScanStateRequest.newBuilder() + .setState(LifecycleScanState.newBuilder().setBucketKey("dummy").setScanStartTime(1L).build()) + .build()) + .build(); + + OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest); + request.setUGI(UserGroupInformation.getCurrentUser()); + + OMException exception = assertThrows(OMException.class, () -> { + request.preExecute(ozoneManager); + }); + + assertEquals(OMException.ResultCodes.ACCESS_DENIED, exception.getResult()); + assertTrue(exception.getMessage().contains("Superuser privilege is required")); + + // Test when user is admin + when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true); + OMRequest preExecuted = request.preExecute(ozoneManager); + assertEquals(omRequest, preExecuted); + + // Test when ACLs are disabled + when(ozoneManager.getAclsEnabled()).thenReturn(false); + when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false); + preExecuted = request.preExecute(ozoneManager); + assertEquals(omRequest, preExecuted); + } + + @Test + public void testValidateAndUpdateCache() throws Exception { + OzoneManager ozoneManager = mock(OzoneManager.class); + OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + + Table table = mock(Table.class); + when(omMetadataManager.getLifecycleScanStateTable()).thenReturn(table); + + LifecycleScanState stateProto = LifecycleScanState.newBuilder() + .setBucketKey("/vol1/bucket1") + .setScanStartTime(123456789L) + .setLastScannedKey("key1") + .build(); + + SaveLifecycleScanStateRequest saveRequest = SaveLifecycleScanStateRequest.newBuilder() + .setState(stateProto) + .build(); + + OMRequest omRequest = OMRequest.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState) + .setClientId(UUID.randomUUID().toString()) + .setSaveLifecycleScanStateRequest(saveRequest) + .build(); + + OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest); + + OMRequest preExecuted = request.preExecute(ozoneManager); + assertEquals(omRequest, preExecuted); + + OMClientResponse response = request.validateAndUpdateCache(ozoneManager, 100L); + assertNotNull(response); + assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus()); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java index 229f4cb459bf..0422aa0e63e6 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponse.java @@ -101,7 +101,7 @@ public void testKeysDeleteResponse() throws Exception { protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse, OmBucketInfo omBucketInfo) { return new OMKeysDeleteResponse( - omResponse, omKeyInfoList, omBucketInfo, Collections.emptyMap()); + omResponse, omKeyInfoList, omBucketInfo, Collections.emptyMap(), null); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java index 1f99c90e4d9a..c34356e2d1cd 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysDeleteResponseWithFSO.java @@ -111,7 +111,7 @@ protected OMClientResponse getOmKeysDeleteResponse(OMResponse omResponse, OmBucketInfo omBucketInfo) { return new OMKeysDeleteResponseWithFSO( omResponse, getOmKeyInfoList(), dirDeleteList, omBucketInfo, - volId, Collections.emptyMap()); + volId, Collections.emptyMap(), null); } @Test diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java new file mode 100644 index 000000000000..4109a9f0f74f --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.response.lifecycle; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.junit.jupiter.api.Test; + +/** + * Tests OMLifecycleSaveScanStateResponse. + */ +public class TestOMLifecycleSaveScanStateResponse { + + @Test + public void testAddToDBBatch() throws Exception { + OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); + BatchOperation batchOperation = mock(BatchOperation.class); + Table table = mock(Table.class); + when(omMetadataManager.getLifecycleScanStateTable()).thenReturn(table); + + OmLifecycleScanState state = new OmLifecycleScanState.Builder() + .setBucketKey("/vol1/bucket1") + .setScanStartTime(123456789L) + .setLastScannedKey("key1") + .build(); + + OMResponse omResponse = OMResponse.newBuilder() + .setCmdType(OzoneManagerProtocolProtos.Type.SaveLifecycleScanState) + .setStatus(OzoneManagerProtocolProtos.Status.OK) + .build(); + + OMLifecycleSaveScanStateResponse response = new OMLifecycleSaveScanStateResponse(omResponse, state); + response.addToDBBatch(omMetadataManager, batchOperation); + + verify(table, times(1)).putWithBatch(batchOperation, "/vol1/bucket1", state); + } +} diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index cd2191633eef..6e741d404f23 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -27,6 +27,10 @@ import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_ENABLED; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_INTERVAL; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT; import static org.apache.hadoop.ozone.om.OmConfig.Keys.ENABLE_FILESYSTEM_PATHS; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_REQUEST; import static org.apache.hadoop.ozone.om.helpers.BucketLayout.FILE_SYSTEM_OPTIMIZED; @@ -44,10 +48,17 @@ import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -71,6 +82,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.hdds.utils.db.DBConfigFromFile; +import org.apache.hadoop.hdds.utils.db.RocksDatabaseException; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -79,6 +91,7 @@ import org.apache.hadoop.ozone.om.FaultInjectorImpl; import org.apache.hadoop.ozone.om.KeyManager; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; import org.apache.hadoop.ozone.om.OmTestManagers; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.OzoneTrash; @@ -99,6 +112,7 @@ import org.apache.hadoop.ozone.om.helpers.OmLCRule; import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration; import org.apache.hadoop.ozone.om.helpers.OmLifecycleRuleAndOperator; +import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.helpers.OmMultipartInfo; import org.apache.hadoop.ozone.om.helpers.OmMultipartKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; @@ -125,6 +139,8 @@ import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; @@ -146,6 +162,8 @@ */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @Timeout(300) +@ParameterizedClass +@MethodSource("stateSaveConfiguration") class TestKeyLifecycleService extends OzoneTestBase { private static final Logger LOG = LoggerFactory.getLogger(TestKeyLifecycleService.class); @@ -168,6 +186,20 @@ class TestKeyLifecycleService extends OzoneTestBase { private KeyLifecycleServiceMetrics metrics; private long bucketObjectID; + @Parameter(0) + private long stateSaveInternal; + + @Parameter(1) + private long maxKeysProcessedPerState; + + static Stream stateSaveConfiguration() { + return Stream.of( + Arguments.of(-1, -1), + Arguments.of(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT, + OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT) + ); + } + @BeforeAll void setup() { ExitUtils.disableSystemExit(); @@ -184,6 +216,8 @@ private void createConfig(File testDir) { conf.setInt(OZONE_KEY_LIFECYCLE_SERVICE_DELETE_BATCH_SIZE, 50); conf.setQuietMode(false); conf.setBoolean(ENABLE_FILESYSTEM_PATHS, false); + conf.setLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, stateSaveInternal); + conf.setLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, maxKeysProcessedPerState); OmLCExpiration.setTest(true); } @@ -221,6 +255,7 @@ void setup(@TempDir File testDir) throws Exception { void resume() { keyLifecycleService.setOzoneTrash(null); keyLifecycleService.setMoveToTrashEnabled(true); + KeyLifecycleService.setInjectors(null); } @AfterAll @@ -251,12 +286,13 @@ void testAllKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); // check there are keys in keyTable assertEquals(KEY_COUNT, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, @@ -265,15 +301,510 @@ void testAllKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); if (createPrefix) { - createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); } else { - OmLCFilter.Builder filter = getOmLCFilterBuilder(prefix, null, null); + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); + } + + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000); + assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount); + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @MethodSource("parameters1") + void testScanStatePiggybackedOnDelete(BucketLayout bucketLayout, boolean createPrefix) throws IOException, + TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key"; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(bucketLayout); + // create keys + List keyList = + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); + // check there are keys in keyTable + assertEquals(KEY_COUNT, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, + WAIT_CHECK_INTERVAL, 1000); + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + + if (createPrefix) { + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); + } else { + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); } + // Wait for deletion GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000); assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount); + + // Verify that scan state was updated through the DeleteKeysRequest + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey); + assertNotNull(scanState); + assertNotNull(scanState.getScanEndTime()); + + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"}) + void testPeriodicStateSave(BucketLayout bucketLayout) throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key"; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(bucketLayout); + long initialNumKeyDeleted = metrics.getNumKeyDeleted().value(); + long initialNumKeyIterated = metrics.getNumKeyIterated().value(); + int testKeyCount = 5; + + // Suspend service so it doesn't process immediately after we create the policy + keyLifecycleService.suspend(); + + // create keys + List keyList = + createKeys(volumeName, bucketName, bucketLayout, testKeyCount, 1, keyPrefix, null); + assertEquals(testKeyCount, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount, + WAIT_CHECK_INTERVAL, 1000); + + // Inject spy to LifecycleScanStateTable to count put operations + Field tableField = OmMetadataManagerImpl.class.getDeclaredField("lifecycleScanStateTable"); + tableField.setAccessible(true); + Table originalTable = + (Table) tableField.get(metadataManager); + Table spyTable = spy(originalTable); + tableField.set(metadataManager, spyTable); + + try { + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); + + // Resume the service + keyLifecycleService.resume(); + + // Wait for deletion + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 10000); + assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount); + + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + + if (stateSaveInternal == -1) { + // With 5 keys and stateSaveIntervalMs = -1, it should save on every key iteration. + // It will save at least 5 times (one for each key). + verify(spyTable, atLeast(5)) + .addCacheEntry(argThat(k -> k.getCacheKey().equals(bucketKey)), any()); + } else { + // With 5 keys and maxKeysProcessedPerState = 100000, there is 1 save piggybacked in KeysDelete request. + verify(spyTable, atLeast(1)) + .addCacheEntry(argThat(k -> k.getCacheKey().equals(bucketKey)), any()); + } + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 5000); + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 5000); + GenericTestUtils.waitFor(() -> + (metrics.getNumKeyDeleted().value() - initialNumKeyDeleted) == testKeyCount, WAIT_CHECK_INTERVAL, 5000); + GenericTestUtils.waitFor(() -> + (metrics.getNumKeyIterated().value() - initialNumKeyIterated) == testKeyCount, WAIT_CHECK_INTERVAL, 5000); + } finally { + deleteLifecyclePolicy(volumeName, bucketName); + } + } + + @ParameterizedTest + @MethodSource("parameters1") + void testBucketScanResume(BucketLayout bucketLayout, boolean createPrefix) throws IOException, + TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key"; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(bucketLayout); + int testKeyCount = 5; + + // Suspend service so it doesn't process immediately after we create the policy + keyLifecycleService.suspend(); + + // create keys + List keyList = + createKeys(volumeName, bucketName, bucketLayout, testKeyCount, 1, keyPrefix, null); + assertEquals(testKeyCount, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount, + WAIT_CHECK_INTERVAL, 1000); + + // determine db keys + List dbKeys = new ArrayList<>(); + long bucketId = + metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID(); + long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID(); + + for (OmKeyArgs args : keyList) { + if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + dbKeys.add(metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, args.getKeyName())); + } else { + dbKeys.add(metadataManager.getOzoneKey(volumeName, bucketName, args.getKeyName())); + } + } + Collections.sort(dbKeys); + String lastScannedDbKey = dbKeys.get(2); // The 3rd key + + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + if (createPrefix) { + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); + } else { + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); + } + + // inject the resume state + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + + OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName); + OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketId) + .setLifecycleConfigurationUpdateID(policy.getUpdateID()) + .setScanStartTime(System.currentTimeMillis()); + + if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + stateBuilder.setLastScannedKey(lastScannedDbKey); + stateBuilder.setLastScannedDir(""); + stateBuilder.setLastScannedDirKey(""); + } else { + stateBuilder.setLastScannedKey(lastScannedDbKey); + } + OmLifecycleScanState scanState = stateBuilder.build(); + metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState); + metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey), + CacheValue.get(1L, scanState)); + OmLifecycleScanState state = metadataManager.getLifecycleScanStateTable().get(bucketKey); + assertNotNull(state); + + // resume the service + keyLifecycleService.resume(); + + // wait for it to process + // it should skip the first 3 keys (index 0, 1, 2) since we set lastScannedDbKey as index 2. + // So it deletes only the last 2 keys (index 3 and 4). + int expectedDeleted = 2; + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 10000); + + // confirm it hasn't deleted all keys + assertEquals(testKeyCount - expectedDeleted, getKeyCount(bucketLayout) - initialKeyCount); + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @MethodSource("parameters1") + void testBucketScanWithScanEndTime(BucketLayout bucketLayout, boolean createPrefix) throws IOException, + TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key"; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(bucketLayout); + int testKeyCount = 5; + + // Suspend service so it doesn't process immediately after we create the policy + keyLifecycleService.suspend(); + + // create keys + List keyList = + createKeys(volumeName, bucketName, bucketLayout, testKeyCount, 1, keyPrefix, null); + assertEquals(testKeyCount, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount, + WAIT_CHECK_INTERVAL, 1000); + + // determine db keys + List dbKeys = new ArrayList<>(); + long bucketId = + metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID(); + long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID(); + + for (OmKeyArgs args : keyList) { + if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + dbKeys.add(metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, args.getKeyName())); + } else { + dbKeys.add(metadataManager.getOzoneKey(volumeName, bucketName, args.getKeyName())); + } + } + Collections.sort(dbKeys); + String lastScannedDbKey = dbKeys.get(2); // The 3rd key + + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + if (createPrefix) { + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); + } else { + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); + } + + // inject the resume state but with ScanEndTime set! + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName); + OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketId) + .setLifecycleConfigurationUpdateID(policy.getUpdateID()) + .setScanStartTime(System.currentTimeMillis()) + .setScanEndTime(System.currentTimeMillis()); // Scan finished previously + + if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) { + stateBuilder.setLastScannedKey(lastScannedDbKey); + stateBuilder.setLastScannedDir(""); + } else { + stateBuilder.setLastScannedKey(lastScannedDbKey); + } + OmLifecycleScanState scanState = stateBuilder.build(); + metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState); + metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey), + CacheValue.get(1L, scanState)); + + // resume the service + keyLifecycleService.resume(); + + // wait for it to process + // Since ScanEndTime is set, it will ignore the lastScannedKey and start from the beginning! + // So it should delete ALL 5 keys. + int expectedDeleted = 5; + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 10000); + + // confirm it has deleted all keys + assertEquals(testKeyCount - expectedDeleted, getKeyCount(bucketLayout) - initialKeyCount); + + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @MethodSource("parameters1") + void testScanEmptyBucket(BucketLayout bucketLayout, boolean moveToTrash) throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + + keyLifecycleService.setMoveToTrashEnabled(moveToTrash); + + // Create empty bucket + createVolumeAndBucket(volumeName, bucketName, bucketLayout, + UserGroupInformation.getCurrentUser().getShortUserName()); + + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + OmLCFilter.Builder filter = bucketLayout == FILE_SYSTEM_OPTIMIZED ? getOmLCFilterBuilder("", null, null) : + getOmLCFilterBuilder("key", null, null); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); + + // Wait until scan completes. Since the bucket is empty, it will scan immediately. + // We check if the scanEndTime is set. + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + + GenericTestUtils.waitFor(() -> { + try { + OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey); + return scanState != null && scanState.getScanEndTime() != null; + } catch (IOException e) { + return false; + } + }, WAIT_CHECK_INTERVAL, 10000); + + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @MethodSource("parameters1") + void testScanStateFailureDoesNotImpactScan(BucketLayout bucketLayout, boolean createPrefix) throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : "key"; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(bucketLayout); + // create keys + List keyList = + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); + assertEquals(KEY_COUNT, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, + WAIT_CHECK_INTERVAL, 1000); + + // Inject failure to LifecycleScanStateTable + Field tableField = OmMetadataManagerImpl.class.getDeclaredField("lifecycleScanStateTable"); + tableField.setAccessible(true); + Table originalTable = + (Table) tableField.get(metadataManager); + Table spyTable = spy(originalTable); + doThrow(new RocksDatabaseException("Injected exception for testing")).when(spyTable).get(any()); + doThrow(new RocksDatabaseException("Injected exception for testing")).when(spyTable).put(any(), any()); + tableField.set(metadataManager, spyTable); + + try { + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + if (createPrefix) { + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); + } else { + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); + } + + // Even though reading scan state fails, the deletion should still proceed normally. + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000); + assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount); + + deleteLifecyclePolicy(volumeName, bucketName); + } finally { + // Restore original table + tableField.set(metadataManager, originalTable); + } + } + + public Stream parameters12() { + return Stream.of( + arguments(FILE_SYSTEM_OPTIMIZED, 2), + arguments(FILE_SYSTEM_OPTIMIZED, 3), + arguments(FILE_SYSTEM_OPTIMIZED, 7), + arguments(BucketLayout.OBJECT_STORE, 2), + arguments(BucketLayout.OBJECT_STORE, 3), + arguments(BucketLayout.OBJECT_STORE, 7) + ); + } + + @ParameterizedTest + @MethodSource("parameters12") + void testNestedFSODirectoryScanResume(BucketLayout bucketLayout, int maxSize) throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String prefix = ""; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(bucketLayout); + long keyIterated = metrics.getNumKeyIterated().value(); + int testKeyCount = 8; + + keyLifecycleService.setListMaxSize(maxSize); + // Suspend service so it doesn't process immediately after we create the policy + keyLifecycleService.suspend(); + + createVolumeAndBucket(volumeName, bucketName, bucketLayout, + UserGroupInformation.getCurrentUser().getShortUserName()); + /** + * Create nested directory and 8 keys inside + * / + * dir1 dir2 dir3 dir30 + * / \ / \ + * dir4 dir5 dir6 dir7 + * / \ + * dir8 dir9 + * + * MaxSize = 2, lastScannedDir is dir9, lastScannedKey is key8 + * MaxSize = 3, lastScannedDir is dir8, lastScannedKey is key6 + * MaxSize = 7, lastScannedDir is dir5, lastScannedKey is key3 + */ + List keyList = new ArrayList<>(); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir4/key0", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir4/key1", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir5/key2", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/dir5/key3", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir8/key5", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir8/key6", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir9/key7", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir3/dir6/dir9/key8", 1, null)); + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + createDirectory(volumeName, bucketName, "dir2"); + createDirectory(volumeName, bucketName, "dir3/dir7"); + createDirectory(volumeName, bucketName, "dir30"); + } + + assertEquals(testKeyCount, keyList.size()); + GenericTestUtils.waitFor( + () -> getKeyCount(bucketLayout) - initialKeyCount == testKeyCount, WAIT_CHECK_INTERVAL, 1000); + + // Create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + + // Inject to cause resume case for FSO with nested directory + FaultInjectorImpl lastFaultInjector = new FaultInjectorImpl(); + lastFaultInjector.setException(new IOException("Injected exception for testing")); + KeyLifecycleService.setInjectors( + Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl(), lastFaultInjector)); + // Resume the service + keyLifecycleService.resume(); + KeyLifecycleService.getInjector(0).resume(); + KeyLifecycleService.getInjector(1).resume(); + + // wait for scanState to be updated + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + GenericTestUtils.waitFor(() -> { + try { + OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey); + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + return scanState != null && scanState.getLastScannedDir() != null && scanState.getLastScannedKey() != null; + } else { + return scanState != null && scanState.getLastScannedKey() != null; + } + } catch (IOException e) { + return false; + } + }, WAIT_CHECK_INTERVAL, 10000); + + OmLifecycleScanState scanState = metadataManager.getLifecycleScanStateTable().get(bucketKey); + if (stateSaveInternal != -1) { + if (maxSize == 2) { + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + assertEquals("dir3/dir6/dir9", scanState.getLastScannedDir()); + assertTrue(scanState.getLastScannedKey().endsWith("key8")); + } else { + assertTrue(scanState.getLastScannedKey().endsWith("key1")); + } + } else if (maxSize == 3) { + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + assertEquals("dir3/dir6/dir8", scanState.getLastScannedDir()); + assertTrue(scanState.getLastScannedKey().endsWith("key6")); + } else { + assertTrue(scanState.getLastScannedKey().endsWith("key2")); + } + } else { + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + assertEquals("dir1/dir5", scanState.getLastScannedDir()); + assertTrue(scanState.getLastScannedKey().endsWith("key3")); + } else { + assertTrue(scanState.getLastScannedKey().endsWith("key7")); + } + } + } + + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == testKeyCount, WAIT_CHECK_INTERVAL, 10000); + // Confirm all keys are deleted + assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount); + // Confirm iterated key number + GenericTestUtils.waitFor(() -> + testKeyCount == metrics.getNumKeyIterated().value() - keyIterated, WAIT_CHECK_INTERVAL, 5000); + deleteLifecyclePolicy(volumeName, bucketName); } @@ -283,12 +814,12 @@ void testOneKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); // check there are keys in keyTable assertEquals(KEY_COUNT, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, @@ -297,16 +828,266 @@ void testOneKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws I ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); int keyIndex = ThreadLocalRandom.current().nextInt(KEY_COUNT - 1); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyList.get(keyIndex).getKeyName(); if (createPrefix) { - createLifecyclePolicy(volumeName, bucketName, bucketLayout, - keyList.get(keyIndex).getKeyName(), null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); } else { - OmLCFilter.Builder filter = getOmLCFilterBuilder(keyList.get(keyIndex).getKeyName(), null, null); + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); } - GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == 1, WAIT_CHECK_INTERVAL, 10000); - assertEquals(KEY_COUNT - 1, getKeyCount(bucketLayout) - initialKeyCount); + int expectedDeleteCount = bucketLayout == FILE_SYSTEM_OPTIMIZED ? KEY_COUNT : 1; + GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == expectedDeleteCount, + WAIT_CHECK_INTERVAL, 5000); + assertEquals(KEY_COUNT - expectedDeleteCount, getKeyCount(bucketLayout) - initialKeyCount); + deleteLifecyclePolicy(volumeName, bucketName); + } + + public Stream parameters13() { + return Stream.of( + arguments("dirC", null, 3), + arguments("dirC/dir3", null, 3), + arguments("dirB", new String[]{"dirC"}, 2), + arguments("dirB/dir2", new String[]{"dirC"}, 2), + arguments("dirA", new String[]{"dirC", "dirB"}, 1), + arguments("dirA/dir1", new String[]{"dirC", "dirB"}, 1) + ); + } + + @ParameterizedTest + @MethodSource("parameters13") + void testDirectorySkippedAfterResume(String lastScannedDir, String[] skippedDir, int expectedDeleted) + throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String prefix = ""; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED); + keyLifecycleService.setListMaxSize(1); + // Suspend service so it doesn't process immediately after we create the policy + keyLifecycleService.suspend(); + + createVolumeAndBucket(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED, + UserGroupInformation.getCurrentUser().getShortUserName()); + + // Create 3 directories: dirA/dir1, dirB/dir2, dirC/dir3 + // Inside each directory, create 1 keys. + int testKeyCount = 3; + List keyList = new ArrayList<>(); + + int i = 0; + for (String dir : Arrays.asList("dirA/dir1", "dirB/dir2", "dirC/dir3")) { + String keyName = dir + "/key" + i++; + keyList.add(createAndCommitKey(volumeName, bucketName, keyName, 1, null)); + } + + assertEquals(testKeyCount, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount == testKeyCount, + WAIT_CHECK_INTERVAL, 1000); + + // Determine DB keys for the files + long bucketId = + metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID(); + long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID(); + + // Find dirB/dir2's objectID and its table key + String dirBKey = metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, lastScannedDir); + + // Create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + createLifecyclePolicy(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED, prefix, + null, date.toString(), true); + + // Inject the resume state for FSO where lastScannedDir is dirB + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName); + OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketId) + .setLifecycleConfigurationUpdateID(policy.getUpdateID()) + .setScanStartTime(System.currentTimeMillis()) + .setLastScannedDir(lastScannedDir) + .setLastScannedDirKey(dirBKey); + + OmLifecycleScanState scanState = stateBuilder.build(); + metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState); + metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey), + CacheValue.get(1L, scanState)); + + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs( + LoggerFactory.getLogger(KeyLifecycleService.class)); + // Resume the service + keyLifecycleService.resume(); + + // Wait for it to process + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) == expectedDeleted, WAIT_CHECK_INTERVAL, 10000); + + // Confirm it hasn't deleted dirA's keys + assertEquals(testKeyCount - expectedDeleted, getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount); + if (skippedDir != null) { + Arrays.stream(skippedDir).forEach( + d -> assertTrue(logCapturer.getOutput().contains("Skip " + d))); + logCapturer.clearOutput(); + } + + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void testLastScannedKeySeek(boolean keyBelongToDir) throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String prefix = ""; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED); + long keyIterated = metrics.getNumKeyIterated().value(); + + keyLifecycleService.suspend(); + + createVolumeAndBucket(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED, + UserGroupInformation.getCurrentUser().getShortUserName()); + + List keyList = new ArrayList<>(); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/key1", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir1/key2", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir2/key3", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "dir2/key4", 1, null)); + + assertEquals(4, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount == 4, + WAIT_CHECK_INTERVAL, 1000); + + long bucketId = + metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID(); + long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID(); + + String dir1Key = metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "dir1"); + long dir1Id = metadataManager.getDirectoryTable().get( + metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "dir1")).getObjectID(); + long dir2Id = metadataManager.getDirectoryTable().get( + metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "dir2")).getObjectID(); + + // key2 is under dir1 + String key2DbKey = metadataManager.getOzonePathKey(volumeId, bucketId, dir1Id, "key2"); + // key3 is under dir2 + String key3DbKey = metadataManager.getOzonePathKey(volumeId, bucketId, dir2Id, "key3"); + + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + OmLCFilter.Builder filter = getOmLCFilterBuilder(prefix, null, null); + createLifecyclePolicy(volumeName, bucketName, BucketLayout.FILE_SYSTEM_OPTIMIZED, null, + filter.build(), date.toString(), true); + + // Set lastScannedDir to dir1, but lastScannedKey to key3 (which is in dir2) + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName); + OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketId) + .setLifecycleConfigurationUpdateID(policy.getUpdateID()) + .setScanStartTime(System.currentTimeMillis()); + + if (keyBelongToDir) { + stateBuilder.setLastScannedDir("dir1").setLastScannedDirKey(dir1Key).setLastScannedKey(key2DbKey); + } else { + stateBuilder.setLastScannedDir("dir1").setLastScannedDirKey(dir1Key).setLastScannedKey(key3DbKey); + } + + OmLifecycleScanState scanState = stateBuilder.build(); + metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState); + metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey), + CacheValue.get(1L, scanState)); + + GenericTestUtils.LogCapturer logCapturer = GenericTestUtils.LogCapturer.captureLogs( + LoggerFactory.getLogger(KeyLifecycleService.class)); + keyLifecycleService.resume(); + + // dir1 can be fully evaluated depending on whether lastScannedKey belong to it (no seek) or not + // dir2 should be skipped dir2 > dir1 + int expectedDeleted = 2; + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 5000); + assertEquals(keyList.size() - expectedDeleted, getKeyCount(BucketLayout.FILE_SYSTEM_OPTIMIZED) - initialKeyCount); + GenericTestUtils.waitFor(() -> + expectedDeleted == metrics.getNumKeyIterated().value() - keyIterated, WAIT_CHECK_INTERVAL, 5000); + if (keyBelongToDir) { + assertTrue(logCapturer.getOutput().contains("Seek to key")); + } else { + assertFalse(logCapturer.getOutput().contains("Seek to key")); + } + deleteLifecyclePolicy(volumeName, bucketName); + } + + @ParameterizedTest + @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"}) + void testBucketRootScannedDirResume(BucketLayout layout) throws Exception { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + String prefix = ""; + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyCount = getKeyCount(layout); + long keyIterated = metrics.getNumKeyIterated().value(); + + keyLifecycleService.suspend(); + + createVolumeAndBucket(volumeName, bucketName, layout, + UserGroupInformation.getCurrentUser().getShortUserName()); + + List keyList = new ArrayList<>(); + keyList.add(createAndCommitKey(volumeName, bucketName, "key1", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "key2", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, "key3", 1, null)); + + assertEquals(3, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(layout) - initialKeyCount == 3, + WAIT_CHECK_INTERVAL, 1000); + + long bucketId = + metadataManager.getBucketTable().get(metadataManager.getBucketKey(volumeName, bucketName)).getObjectID(); + long volumeId = metadataManager.getVolumeTable().get(metadataManager.getVolumeKey(volumeName)).getObjectID(); + + // key2 in bucket root + String key2DbKey = layout == FILE_SYSTEM_OPTIMIZED ? + metadataManager.getOzonePathKey(volumeId, bucketId, bucketId, "key2") : + metadataManager.getOzoneKey(volumeName, bucketName, "key2"); + + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + OmLCFilter.Builder filter = getOmLCFilterBuilder(prefix, null, null); + createLifecyclePolicy(volumeName, bucketName, layout, null, + filter.build(), date.toString(), true); + + // Set lastScannedDir to "" (bucket root) and lastScannedKey to key2 + String bucketKey = metadataManager.getBucketKey(volumeName, bucketName); + OmLifecycleConfiguration policy = metadataManager.getLifecycleConfiguration(volumeName, bucketName); + OmLifecycleScanState.Builder stateBuilder = new OmLifecycleScanState.Builder() + .setBucketKey(bucketKey) + .setBucketObjID(bucketId) + .setLifecycleConfigurationUpdateID(policy.getUpdateID()) + .setScanStartTime(System.currentTimeMillis()) + .setLastScannedDir("") + .setLastScannedDirKey("") + .setLastScannedKey(key2DbKey); + + OmLifecycleScanState scanState = stateBuilder.build(); + metadataManager.getLifecycleScanStateTable().put(bucketKey, scanState); + metadataManager.getLifecycleScanStateTable().addCacheEntry(new CacheKey<>(bucketKey), + CacheValue.get(1L, scanState)); + + keyLifecycleService.resume(); + + // It should seek to key2. key1 is skipped, key2 is skipped too. + // So key1 and key2 are skipped, key3 is deleted. + int expectedDeleted = 1; + GenericTestUtils.waitFor(() -> + (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 10000); + + assertEquals(2, getKeyCount(layout) - initialKeyCount); + GenericTestUtils.waitFor(() -> + expectedDeleted == metrics.getNumKeyIterated().value() - keyIterated, WAIT_CHECK_INTERVAL, 5000); deleteLifecyclePolicy(volumeName, bucketName); } @@ -328,11 +1109,11 @@ void testOnlyKeyExpired(BucketLayout bucketLayout, boolean createPrefix) throws // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyArg.getKeyName(); if (createPrefix) { - createLifecyclePolicy(volumeName, bucketName, bucketLayout, - keyArg.getKeyName(), null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); } else { - OmLCFilter.Builder filter = getOmLCFilterBuilder(keyArg.getKeyName(), null, null); + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); } @@ -414,13 +1195,13 @@ void testAllKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); Map tags = ImmutableMap.of("app", "spark", "user", "ozone"); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, tags); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, tags); // check there are keys in keyTable assertEquals(KEY_COUNT, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, @@ -428,7 +1209,8 @@ void testAllKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(prefix, tags).build(); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; + OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(rulePrefix, tags).build(); OmLCFilter.Builder filter = getOmLCFilterBuilder(null, null, andOperator); createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); @@ -445,15 +1227,15 @@ void testOneKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept int keyCount = KEY_COUNT; final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); Map tags = ImmutableMap.of("app", "spark", "user", "ozone"); // create keys without tags List keyList = - createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, keyPrefix, null); // create one more key with tag - final String keyName = uniqueObjectName(prefix); + final String keyName = uniqueObjectName(keyPrefix); // Create the key OmKeyArgs keyArg = createAndCommitKey(volumeName, bucketName, keyName, 1, tags); keyList.add(keyArg); @@ -466,7 +1248,8 @@ void testOneKeyExpiredWithAndOperator(BucketLayout bucketLayout) throws IOExcept // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(prefix, tags).build(); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; + OmLifecycleRuleAndOperator andOperator = getOmLCAndOperatorBuilder(rulePrefix, tags).build(); OmLCFilter.Builder filter = getOmLCFilterBuilder(null, null, andOperator); createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); @@ -573,10 +1356,12 @@ void testRootSlashPrefix(BucketLayout bucketLayout, String prefix) @MethodSource("parameters1") void testSlashPrefix(BucketLayout bucketLayout, boolean createPrefix) throws IOException, TimeoutException, InterruptedException { - assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED); + // FSO bucket must end with "/". "/" is also invalid prefix for FSO. + assumeTrue(bucketLayout == OBJECT_STORE); final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); String keyPrefix = "key"; + String rulePrefix = "/" + keyPrefix; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys @@ -590,9 +1375,9 @@ void testSlashPrefix(BucketLayout bucketLayout, boolean createPrefix) ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); if (createPrefix) { - createLifecyclePolicy(volumeName, bucketName, bucketLayout, "/" + keyPrefix, null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); } else { - OmLCFilter.Builder filter = getOmLCFilterBuilder("/" + keyPrefix, null, null); + OmLCFilter.Builder filter = getOmLCFilterBuilder(rulePrefix, null, null); createLifecyclePolicy(volumeName, bucketName, bucketLayout, null, filter.build(), date.toString(), true); } @@ -612,6 +1397,7 @@ void testSlashPrefix(BucketLayout bucketLayout, boolean createPrefix) @MethodSource("parameters1") void testSlashKey(BucketLayout bucketLayout, boolean createPrefix) throws IOException, TimeoutException, InterruptedException { + // FSO bucket doesn't allow "//" in prefix. assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED); final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); @@ -645,6 +1431,7 @@ void testSlashKey(BucketLayout bucketLayout, boolean createPrefix) @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"}) void testSlashKeyWithAndOperator(BucketLayout bucketLayout) throws IOException, TimeoutException, InterruptedException { + // FSO bucket doesn't allow "//" in prefix. assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED); final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); @@ -676,6 +1463,7 @@ void testSlashKeyWithAndOperator(BucketLayout bucketLayout) @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"}) void testSlashPrefixWithAndOperator(BucketLayout bucketLayout) throws IOException, InterruptedException, TimeoutException { + // FSO bucket must end with "/". "/" is also invalid prefix for FSO. assumeTrue(bucketLayout != FILE_SYSTEM_OPTIMIZED); final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); @@ -709,14 +1497,14 @@ void testComplexPrefix(BucketLayout bucketLayout) throws IOException, TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "dir1/dir2/dir3/key"; + String keyPrefix = "dir1/dir2/dir3/key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); long initialNumDeletedKey = metrics.getNumKeyDeleted().value(); long initialSizeDeletedKey = metrics.getSizeKeyDeleted().value(); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); // check there are keys in keyTable assertEquals(KEY_COUNT, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, @@ -724,7 +1512,8 @@ void testComplexPrefix(BucketLayout bucketLayout) throws IOException, // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "dir1/dir2/dir3/" : keyPrefix; + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000); @@ -741,7 +1530,7 @@ void testPrefixNotMatch(BucketLayout bucketLayout) throws IOException, Interrupt final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); String keyPrefix = "dir1/dir2/dir3/key"; - String filterPrefix = "dir1/dir2/dir4/key"; + String filterPrefix = "dir1/dir2/dir4/"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys @@ -769,12 +1558,12 @@ void testExpireKeysUnderDirectory(BucketLayout bucketLayout) throws IOException, TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "dir1/dir2/dir3/key"; + String keyPrefix = "dir1/dir2/dir3/key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); // check there are keys in keyTable assertEquals(KEY_COUNT, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, @@ -788,7 +1577,8 @@ void testExpireKeysUnderDirectory(BucketLayout bucketLayout) throws IOException, // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "dir1/dir2/dir3/" : keyPrefix; + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); GenericTestUtils.waitFor(() -> (getDeletedKeyCount() - initialDeletedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 10000); @@ -799,8 +1589,6 @@ void testExpireKeysUnderDirectory(BucketLayout bucketLayout) throws IOException, public Stream parameters3() { return Stream.of( arguments("dir1/dir2/dir3/key", "dir1/dir2/dir3/", "dir1/dir2/dir3"), - arguments("dir1/dir2/dir3/key", "dir1/dir2/dir3", "dir1/dir2/dir3"), - arguments("dir1/key", "dir1", "dir1"), arguments("dir1/key", "dir1/", "dir1")); } @@ -856,50 +1644,19 @@ void testMatchedDirectoryNotDeleted(String keyPrefix, String rulePrefix, String public Stream parameters4() { return Stream.of( - arguments("dir1/dir2/dir3", "dir1/dir2/dir3", 3, 0, true, false), - arguments("dir1/dir2/dir3", "dir1/dir2/dir3", 3, 0, false, true), - arguments("/dir1/dir2/dir3", "dir1/dir2/dir3", 3, 1, true, false), - arguments("/dir1/dir2/dir3", "dir1/dir2/dir3", 3, 1, false, true), - arguments("/dir1/dir2/dir3/", "dir1/dir2/dir3/", 3, 0, true, false), - arguments("/dir1/dir2/dir3/", "dir1/dir2/dir3/", 3, 0, false, true), - arguments("/dir1//dir2//dir3//", "dir1/dir2/dir3/", 3, 0, true, false), - arguments("/dir1//dir2//dir3//", "dir1/dir2/dir3/", 3, 0, false, true), - arguments("/dir1//dir2/dir3/", "dir1/dir2/dir", 3, 1, true, false), - arguments("/dir1//dir2/dir3/", "dir1/dir2/dir", 3, 1, false, true), - arguments("/dir1//dir2/dir3/", "dir1/dir2/dir/", 3, 0, true, false), - arguments("/dir1//dir2/dir3/", "dir1/dir2/dir/", 3, 0, false, true), - arguments("dir1/dir2", "dir1/dir2/", 2, 0, true, false), - arguments("dir1/dir2", "dir1/dir2/", 2, 0, false, true), - arguments("/dir1/dir2", "dir1/dir2", 2, 1, true, false), - arguments("/dir1/dir2", "dir1/dir2", 2, 1, false, true), - arguments("/dir1/dir2/", "dir1/dir2", 2, 1, true, false), - arguments("/dir1/dir2/", "dir1/dir2", 2, 1, false, true), - arguments("/dir1//dir2//", "dir1/dir2/", 2, 0, true, false), - arguments("/dir1//dir2//", "dir1/dir2/", 2, 0, false, true), - arguments("/dir1//dir2//", "dir1/dir", 2, 1, true, false), - arguments("/dir1//dir2//", "dir1/dir", 2, 1, false, true), - arguments("/dir1//dir2//", "dir1/dir/", 2, 0, true, false), - arguments("/dir1//dir2//", "dir1/dir/", 2, 0, false, true), - arguments("dir1", "dir1/", 1, 0, true, false), - arguments("dir1", "dir1/", 1, 0, false, true), - arguments("/dir1", "dir1", 1, 1, true, false), - arguments("/dir1", "dir1", 1, 1, false, true), - arguments("/dir1/", "dir1", 1, 1, true, false), - arguments("/dir1/", "dir1", 1, 1, false, true), - arguments("/dir1//", "dir1/", 1, 0, true, false), - arguments("/dir1//", "dir1/", 1, 0, false, true), - arguments("/dir1//", "dir", 1, 1, true, false), - arguments("/dir1//", "dir", 1, 1, false, true), - arguments("/dir1//", "dir/", 1, 0, true, false), - arguments("/dir1//", "dir/", 1, 0, false, true) + arguments("dir1/dir2/dir3//", "dir1/dir2/dir3/", 3, true, false), + arguments("dir1/dir2/dir3//", "dir1/dir2/dir3/", 3, false, true), + arguments("dir1/dir2//", "dir1/dir2/", 2, true, false), + arguments("dir1/dir2//", "dir1/dir2/", 2, false, true), + arguments("dir1//", "dir1/", 1, true, false), + arguments("dir1//", "dir1/", 1, false, true) ); } @ParameterizedTest @MethodSource("parameters4") - void testExpireOnlyDirectory(String dirName, String prefix, int dirDepth, int deletedDirCount, - boolean createPrefix, boolean createFilterPrefix) throws IOException, - TimeoutException, InterruptedException { + void testPrefixDirectoryNotExpired(String dirName, String prefix, int dirDepth, boolean createPrefix, + boolean createFilterPrefix) throws IOException, TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); long initialDeletedDirCount = getDeletedDirectoryCount(); @@ -916,6 +1673,10 @@ void testExpireOnlyDirectory(String dirName, String prefix, int dirDepth, int de assertEquals(dirDepth, getDirCount() - initialDirCount); assertEquals(0, getDeletedDirectoryCount() - initialDeletedDirCount); + GenericTestUtils.LogCapturer log = + GenericTestUtils.LogCapturer.captureLogs( + LoggerFactory.getLogger(KeyLifecycleService.class)); + // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); @@ -928,13 +1689,171 @@ void testExpireOnlyDirectory(String dirName, String prefix, int dirDepth, int de null, filter.build(), date.toString(), true); } - GenericTestUtils.waitFor( - () -> (getDeletedDirectoryCount() - initialDeletedDirCount) == deletedDirCount, WAIT_CHECK_INTERVAL, 10000); - assertEquals(dirDepth - deletedDirCount, getDirCount() - initialDirCount); - assertEquals(deletedDirCount, metrics.getNumDirDeleted().value() - initialNumDeletedDir); + GenericTestUtils.waitFor(() -> log.getOutput().contains("Prefix directory " + prefix + " doesn't get expired"), + WAIT_CHECK_INTERVAL, 10000); + assertEquals(dirDepth, getDirCount() - initialDirCount); + assertEquals(0, metrics.getNumDirDeleted().value() - initialNumDeletedDir); + deleteLifecyclePolicy(volumeName, bucketName); + } + + @Test + void testConsolidatedPrefixNotHappen() throws IOException, TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + long initialDeletedKeyCount = getDeletedKeyCount(); + String dir1 = "dir/dir1/dir2/"; + String dir2 = "log/log1/log2/"; + + // Create the directories + createVolumeAndBucket(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, + UserGroupInformation.getCurrentUser().getShortUserName()); + createDirectory(volumeName, bucketName, dir1); + createDirectory(volumeName, bucketName, dir2); + KeyInfoWithVolumeContext keyInfo = getDirectory(volumeName, bucketName, dir1); + assertFalse(keyInfo.getKeyInfo().isFile()); + keyInfo = getDirectory(volumeName, bucketName, dir2); + assertFalse(keyInfo.getKeyInfo().isFile()); + List keyList = new ArrayList<>(); + keyList.add(createAndCommitKey(volumeName, bucketName, dir1 + "key1", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, dir2 + "key2", 1, null)); + + Thread.sleep(SERVICE_INTERVAL); + + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + List ruleList = new ArrayList<>(); + String ruleID1 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()); + String ruleID2 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()); + ruleList.add(new OmLCRule.Builder().setId(ruleID1) + .setEnabled(true).setPrefix(dir1) + .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) + .build()); + ruleList.add(new OmLCRule.Builder().setId(ruleID2) + .setEnabled(true).setPrefix(dir2) + .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) + .build()); + createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ruleList); + + GenericTestUtils.waitFor(() -> getDeletedKeyCount() - initialDeletedKeyCount == keyList.size(), + WAIT_CHECK_INTERVAL, 5000); deleteLifecyclePolicy(volumeName, bucketName); } + public Stream parameters41() { + return Stream.of( + arguments("dir/dir1/", "dir/dir1/dir2/", null, null, true, + "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/dir1/", null), + arguments("dir/dir1/", "dir/dir2/", null, null, false, + "Prefix directory dir/dir2/ doesn't get expired", 2, "dir/dir2/", null), + arguments("dir1/dir2/", "log1/log2/", null, null, false, + "Prefix directory dir1/dir2/ doesn't get expired", 2, "log1/log2/", null), + arguments("dir/dir1/", "dir/dir1/dir2/", "dir/", null, true, + "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/", null), + arguments("dir/dir1/dir2/", "dir/", "dir/dir1/", null, true, + "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/", null), + arguments("dir/", "dir/dir1/", "dir/dir1/dir2/", null, true, + "Prefix directory dir/dir1/dir2/ doesn't get expired", 1, "dir/", null), + arguments("dir/dir1/", "log/log1/", "dir/dir1/dir2/", null, true, + "Prefix directory dir/dir1/dir2/ doesn't get expired", 2, "log/log1/", "dir/dir1/"), + arguments("dir/dir1/", "log/log1/", "data/data1/", "log/log1/log2/", true, + "Prefix directory data/data1/ doesn't get expired", 3, "log/log1/", "data/data1/") + ); + } + + @ParameterizedTest + @MethodSource("parameters41") + @SuppressWarnings("parameternumber") + void testConsolidatedPrefixDirectoryNotExpired(String dir1, String dir2, String dir3, String dir4, + boolean shouldConsolidateRule, String expectedLog, int consolidatedRuleListSize, + String firstConsolidatedPrefix, String lastConsolidatedPrefix) + throws IOException, TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + long initialDeletedKeyCount = getDeletedKeyCount(); + long initialKeyIterated = metrics.getNumKeyIterated().value(); + long initialKeyDeleted = metrics.getNumKeyDeleted().value(); + + // Create the directories + createVolumeAndBucket(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, + UserGroupInformation.getCurrentUser().getShortUserName()); + createDirectory(volumeName, bucketName, dir1); + createDirectory(volumeName, bucketName, dir2); + KeyInfoWithVolumeContext keyInfo = getDirectory(volumeName, bucketName, dir1); + assertFalse(keyInfo.getKeyInfo().isFile()); + keyInfo = getDirectory(volumeName, bucketName, dir2); + assertFalse(keyInfo.getKeyInfo().isFile()); + List keyList = new ArrayList<>(); + keyList.add(createAndCommitKey(volumeName, bucketName, dir1 + "key1", 1, null)); + keyList.add(createAndCommitKey(volumeName, bucketName, dir2 + "key2", 1, null)); + + Thread.sleep(SERVICE_INTERVAL); + KeyLifecycleService.setTest(true); + KeyLifecycleService.reSetConsolidatedRuleList(); + + GenericTestUtils.LogCapturer log = + GenericTestUtils.LogCapturer.captureLogs( + LoggerFactory.getLogger(KeyLifecycleService.class)); + + // create Lifecycle configuration + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + List ruleList = new ArrayList<>(); + String ruleID1 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()); + String ruleID2 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()); + ruleList.add(new OmLCRule.Builder().setId(ruleID1) + .setEnabled(true).setPrefix(dir1) + .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) + .build()); + ruleList.add(new OmLCRule.Builder().setId(ruleID2) + .setEnabled(true).setPrefix(dir2) + .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) + .build()); + if (dir3 != null) { + String ruleID3 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()); + ruleList.add(new OmLCRule.Builder().setId(ruleID3) + .setEnabled(true).setPrefix(dir3) + .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) + .build()); + keyList.add(createAndCommitKey(volumeName, bucketName, dir3 + "key3", 1, null)); + } + if (dir4 != null) { + String ruleID4 = String.valueOf(OBJECT_ID_COUNTER.getAndIncrement()); + ruleList.add(new OmLCRule.Builder().setId(ruleID4) + .setEnabled(true).setPrefix(dir4) + .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) + .build()); + keyList.add(createAndCommitKey(volumeName, bucketName, dir4 + "key4", 1, null)); + } + createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ruleList); + + try { + if (shouldConsolidateRule) { + GenericTestUtils.waitFor(() -> log.getOutput().contains("Consolidate"), WAIT_CHECK_INTERVAL, 5000); + } + if (expectedLog != null) { + GenericTestUtils.waitFor(() -> log.getOutput().contains(expectedLog), WAIT_CHECK_INTERVAL, 5000); + } + GenericTestUtils.waitFor(() -> getDeletedKeyCount() - initialDeletedKeyCount == keyList.size(), + WAIT_CHECK_INTERVAL, 5000); + GenericTestUtils.waitFor(() -> keyList.size() == metrics.getNumKeyIterated().value() - initialKeyIterated, + WAIT_CHECK_INTERVAL, 5000); + GenericTestUtils.waitFor(() -> keyList.size() == metrics.getNumKeyDeleted().value() - initialKeyDeleted, + WAIT_CHECK_INTERVAL, 5000); + GenericTestUtils.waitFor(() -> { + List list = KeyLifecycleService.getConsolidatedRuleList(); + boolean sizeMatch = list != null && list.size() == consolidatedRuleListSize; + boolean firstPrefixMatch = list != null && + firstConsolidatedPrefix.equals(list.get(0).getConsolidatedPrefix()); + boolean lastPrefixMatch = lastConsolidatedPrefix == null ? true : + list != null && lastConsolidatedPrefix.equals(list.get(list.size() - 1).getConsolidatedPrefix()); + return sizeMatch && firstPrefixMatch && lastPrefixMatch; + }, WAIT_CHECK_INTERVAL, 5000); + } finally { + deleteLifecyclePolicy(volumeName, bucketName); + } + } + @ParameterizedTest @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"}) void testExpireNonExistDirectory(BucketLayout bucketLayout) @@ -962,7 +1881,7 @@ void testExpireNonExistDirectory(BucketLayout bucketLayout) // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, "dir1/dir2/dir4", null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, "dir1/dir2/dir4/", null, date.toString(), true); Thread.sleep(EXPIRE_SECONDS); @@ -1007,12 +1926,12 @@ void testOneRuleDisabledOneRuleEnabled(BucketLayout bucketLayout) throws IOException, InterruptedException, TimeoutException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); // check there are keys in keyTable assertEquals(KEY_COUNT, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, @@ -1021,12 +1940,13 @@ void testOneRuleDisabledOneRuleEnabled(BucketLayout bucketLayout) ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); List ruleList = new ArrayList<>(); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement())) - .setEnabled(false).setPrefix(prefix) + .setEnabled(false).setPrefix(rulePrefix) .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) .build()); ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement())) - .setEnabled(true).setPrefix(prefix) + .setEnabled(true).setPrefix(rulePrefix) .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) .build()); createLifecyclePolicy(volumeName, bucketName, bucketLayout, ruleList); @@ -1041,6 +1961,7 @@ void testOneRuleDisabledOneRuleEnabled(BucketLayout bucketLayout) @ValueSource(strings = {"FILE_SYSTEM_OPTIMIZED", "OBJECT_STORE"}) void testKeyUpdatedShouldNotGetDeleted(BucketLayout bucketLayout) throws IOException, InterruptedException, TimeoutException { + assumeTrue(stateSaveInternal != -1); final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); GenericTestUtils.LogCapturer log = @@ -1049,12 +1970,13 @@ void testKeyUpdatedShouldNotGetDeleted(BucketLayout bucketLayout) GenericTestUtils.LogCapturer requestLog = GenericTestUtils.LogCapturer.captureLogs( LoggerFactory.getLogger(OMKeysDeleteRequest.class)); - String prefix = "key"; + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); KeyLifecycleService.setInjectors( Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl())); @@ -1062,7 +1984,7 @@ void testKeyUpdatedShouldNotGetDeleted(BucketLayout bucketLayout) // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); Thread.sleep(SERVICE_INTERVAL); KeyLifecycleService.getInjector(0).resume(); @@ -1130,7 +2052,7 @@ void testPerformanceWithExpiredKeys(BucketLayout bucketLayout) throws IOException, InterruptedException, TimeoutException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); long initialKeyDeleted = metrics.getNumKeyDeleted().value(); @@ -1139,7 +2061,7 @@ void testPerformanceWithExpiredKeys(BucketLayout bucketLayout) final int keyCount = 10; // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, keyPrefix, null); // check there are keys in keyTable assertEquals(keyCount, keyList.size()); GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == keyCount, @@ -1148,12 +2070,13 @@ void testPerformanceWithExpiredKeys(BucketLayout bucketLayout) ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); List ruleList = new ArrayList<>(); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement())) - .setEnabled(false).setPrefix(prefix) + .setEnabled(false).setPrefix(rulePrefix) .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) .build()); ruleList.add(new OmLCRule.Builder().setId(String.valueOf(OBJECT_ID_COUNTER.getAndIncrement())) - .setEnabled(true).setPrefix(prefix) + .setEnabled(true).setPrefix(rulePrefix) .setAction(new OmLCExpiration.Builder().setDate(date.toString()).build()) .build()); createLifecyclePolicy(volumeName, bucketName, bucketLayout, ruleList); @@ -1251,9 +2174,8 @@ void testPerformanceWithNestedDir(BucketLayout bucketLayout, String prefix) GenericTestUtils.waitFor(() -> metrics.getNumKeyDeleted().value() - initialKeyDeleted == keyCount, WAIT_CHECK_INTERVAL, 5000); assertEquals(0, metrics.getNumDirIterated().value() - initialDirIterated); - GenericTestUtils.waitFor(() -> - metrics.getNumDirDeleted().value() - initialDirDeleted == (bucketLayout == FILE_SYSTEM_OPTIMIZED ? 1 : 0), - WAIT_CHECK_INTERVAL, 10000); + GenericTestUtils.waitFor(() -> metrics.getNumDirDeleted().value() - initialDirDeleted == 0, + WAIT_CHECK_INTERVAL, 5000); deleteLifecyclePolicy(volumeName, bucketName); } @@ -1273,7 +2195,7 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx TimeoutException, InterruptedException { final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); - String prefix = "key"; + String keyPrefix = "key"; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); long initialRenamedKeyCount = metrics.getNumKeyRenamed().value(); @@ -1282,7 +2204,7 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx keyLifecycleService.setListMaxSize(maxListSize); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, keyCount, 1, keyPrefix, null); // check there are keys in keyTable Thread.sleep(SERVICE_INTERVAL); assertEquals(keyCount, keyList.size()); @@ -1305,7 +2227,8 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); if (enableTrash && bucketLayout != OBJECT_STORE) { GenericTestUtils.waitFor(() -> @@ -1316,9 +2239,11 @@ void testListMaxSize(BucketLayout bucketLayout, boolean enableTrash) throws IOEx (getDeletedKeyCount() - initialDeletedKeyCount) == keyCount, WAIT_CHECK_INTERVAL, 5000); assertEquals(0, getKeyCount(bucketLayout) - initialKeyCount); } - GenericTestUtils.waitFor(() -> - log.getOutput().contains("LimitedSizeList has reached maximum size " + maxListSize), - WAIT_CHECK_INTERVAL, 5000); + if (stateSaveInternal != -1) { + GenericTestUtils.waitFor(() -> + log.getOutput().contains("LimitedSizeList has reached maximum size " + maxListSize), + WAIT_CHECK_INTERVAL, 5000); + } GenericTestUtils.setLogLevel(KeyLifecycleService.getLog(), Level.INFO); deleteLifecyclePolicy(volumeName, bucketName); } @@ -1364,67 +2289,98 @@ void testMoveToTrash(BucketLayout bucketLayout, String prefix) throws IOExceptio ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); createLifecyclePolicy(volumeName, bucketName, bucketLayout, "", null, date.toString(), true); - GenericTestUtils.waitFor(() -> - (metrics.getNumKeyRenamed().value() - initialRenamedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 50000); - assertEquals(0, getDeletedKeyCount() - initialDeletedKeyCount); - if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { - // Legacy bucket doesn't have dir concept + try { GenericTestUtils.waitFor(() -> - metrics.getNumDirRenamed().value() - initialRenamedDirCount == (prefix.contains(OM_KEY_PREFIX) ? 1 : 0), - WAIT_CHECK_INTERVAL, 5000); - } - deleteLifecyclePolicy(volumeName, bucketName); - // verify trash directory has the right native ACLs - List dirList = new ArrayList<>(); - if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { - dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX)); - dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner)); - dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + - OM_KEY_PREFIX + CURRENT)); - } else { - dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX)); - dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + OM_KEY_PREFIX)); - dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + - OM_KEY_PREFIX + CURRENT + OM_KEY_PREFIX)); - } - for (KeyInfoWithVolumeContext dir : dirList) { - List aclList = dir.getKeyInfo().getAcls(); - for (OzoneAcl acl : aclList) { - if (acl.getType() == IAccessAuthorizer.ACLIdentityType.USER || - acl.getType() == IAccessAuthorizer.ACLIdentityType.GROUP) { - assertEquals(bucketOwner, acl.getName()); - assertTrue(acl.getAclList().contains(ALL)); + (metrics.getNumKeyRenamed().value() - initialRenamedKeyCount) == KEY_COUNT, WAIT_CHECK_INTERVAL, 5000); + assertEquals(0, getDeletedKeyCount() - initialDeletedKeyCount); + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + // Legacy bucket doesn't have dir concept + GenericTestUtils.waitFor(() -> + metrics.getNumDirRenamed().value() - initialRenamedDirCount == (prefix.contains(OM_KEY_PREFIX) ? + 1 : 0), WAIT_CHECK_INTERVAL, 5000); + } + + // verify that trash directory has the right native ACLs + List dirList = new ArrayList<>(); + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX)); + dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner)); + dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + + OM_KEY_PREFIX + CURRENT)); + } else { + dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX)); + dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + OM_KEY_PREFIX)); + dirList.add(getDirectory(volumeName, bucketName, TRASH_PREFIX + OM_KEY_PREFIX + bucketOwner + + OM_KEY_PREFIX + CURRENT + OM_KEY_PREFIX)); + } + for (KeyInfoWithVolumeContext dir : dirList) { + List aclList = dir.getKeyInfo().getAcls(); + for (OzoneAcl acl : aclList) { + if (acl.getType() == IAccessAuthorizer.ACLIdentityType.USER || + acl.getType() == IAccessAuthorizer.ACLIdentityType.GROUP) { + assertEquals(bucketOwner, acl.getName()); + assertTrue(acl.getAclList().contains(ALL)); + } } } + + // keys under trash directory is counted in getKeyCount() + if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { + assertEquals(KEY_COUNT, getKeyCount(bucketLayout) - initialKeyCount); + } else { + // For legacy bucket, trash directories along .Trash/user-test/Current are in key table too. + assertEquals(KEY_COUNT + (prefix.contains(OM_KEY_PREFIX) ? 4 : 3), + getKeyCount(bucketLayout) - initialKeyCount); + } + } finally { + deleteLifecyclePolicy(volumeName, bucketName); } + } - GenericTestUtils.LogCapturer log = - GenericTestUtils.LogCapturer.captureLogs( - LoggerFactory.getLogger(KeyLifecycleService.class)); + @ParameterizedTest + @MethodSource("parameters7") + void testMoveToTrashWithTrashPrefix(BucketLayout bucketLayout, String prefix) throws IOException, + TimeoutException, InterruptedException { + final String volumeName = getTestName(); + final String bucketName = uniqueObjectName("bucket"); + long initialKeyCount = getKeyCount(bucketLayout); + // create keys + String bucketOwner = UserGroupInformation.getCurrentUser().getShortUserName() + "-test"; + List keyList = + createKeys(volumeName, bucketName, bucketLayout, bucketOwner, KEY_COUNT, 1, prefix, null); + // check there are keys in keyTable + Thread.sleep(SERVICE_INTERVAL); + assertEquals(KEY_COUNT, keyList.size()); + GenericTestUtils.waitFor(() -> getKeyCount(bucketLayout) - initialKeyCount == KEY_COUNT, + WAIT_CHECK_INTERVAL, 1000); - // keys under trash directory is counted in getKeyCount() - if (bucketLayout == FILE_SYSTEM_OPTIMIZED) { - assertEquals(KEY_COUNT, getKeyCount(bucketLayout) - initialKeyCount); - } else { - // For legacy bucket, trash directories along .Trash/user-test/Current are in key table too. - assertEquals(KEY_COUNT + (prefix.contains(OM_KEY_PREFIX) ? 4 : 3), getKeyCount(bucketLayout) - initialKeyCount); - } - // create new policy to test rule with prefix ".Trash/" is ignored during lifecycle evaluation - now = ZonedDateTime.now(ZoneOffset.UTC); + // enabled trash + final float trashInterval = 0.5f; // 30 seconds, 0.5 * (60 * 1000) ms + conf.setFloat(FS_TRASH_INTERVAL_KEY, trashInterval); + FileSystem fs = SecurityUtil.doAsLoginUser( + (PrivilegedExceptionAction) + () -> new TrashOzoneFileSystem(om)); + keyLifecycleService.setOzoneTrash(new OzoneTrash(fs, conf, om)); + + // create a new policy to test rule with prefix ".Trash/" is ignored during lifecycle evaluation + ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); final String expiredDate = now.plusSeconds(EXPIRE_SECONDS).toString(); assertThrowsExactly(OMException.class, () -> createLifecyclePolicy( volumeName, bucketName, bucketLayout, TRASH_PREFIX + OM_KEY_PREFIX, null, expiredDate, true)); - // create new policy to test rule with prefix ".Trash" is ignored during lifecycle evaluation + // create a new policy to test rule with prefix ".Trash" is ignored during lifecycle evaluation assertThrowsExactly(OMException.class, () -> createLifecyclePolicy( volumeName, bucketName, bucketLayout, TRASH_PREFIX, null, expiredDate, true)); - // create new policy to test rule with prefix ".Tras" is ignored during lifecycle evaluation + // create a new policy to test rule with prefix ".Tras/" is ignored during lifecycle evaluation now = ZonedDateTime.now(ZoneOffset.UTC); - date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ".Tras", null, date.toString(), true); + ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); + createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, ".Tras/", null, date.toString(), true); + GenericTestUtils.LogCapturer log = + GenericTestUtils.LogCapturer.captureLogs( + LoggerFactory.getLogger(KeyLifecycleService.class)); GenericTestUtils.waitFor( () -> log.getOutput().contains("No expired keys/dirs found/remained for bucket"), WAIT_CHECK_INTERVAL, 5000); deleteLifecyclePolicy(volumeName, bucketName); @@ -1442,23 +2398,27 @@ void testMoveToTrash(BucketLayout bucketLayout, String prefix) throws IOExceptio public Stream parameters8() { return Stream.of( - arguments("dir1/dir2/dir3/key", null, "dir1/dir", "dir1/dir2/dir3", KEY_COUNT, 2, false), - arguments("dir1/dir2/dir3/key", null, "dir1/dir", "dir1/dir2/dir3", KEY_COUNT, 0, true), - arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/dir", "dir1/dir2/dir3", KEY_COUNT * 2, 4, false), - arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/dir", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true), - arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", KEY_COUNT, 2, false), - arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", KEY_COUNT, 0, true), - arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2", "dir1/dir2/dir3", - KEY_COUNT * 2, 4, false), - arguments("dir1/dir2/dir3/key", "dir1/dir22/dir5/key", "dir1/dir2", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true), - arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir", "dir1/dir2/dir3", KEY_COUNT * 2, 5, false), - arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true), - arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT, 3, false), - arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT, 0, true), - arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1", "dir1/dir2/dir3", KEY_COUNT * 2, 6, false), - arguments("dir1/dir2/dir3/key", "dir11/dir4/dir5/key", "dir1", "dir1/dir2/dir3", KEY_COUNT * 2, 3, true), + // dir3 and keys under dir3 deleted + arguments("dir1/dir2/dir3/key", null, "dir1/dir2/", "dir1/dir2/dir3/", KEY_COUNT, 1, false), + // no dir, but keys under dir3 deleted + arguments("dir1/dir2/dir3/key", null, "dir1/dir2/", "dir1/dir2/dir3/", KEY_COUNT, 0, true), + // dir3 dir5, and all keys under dir3 and dir5 deleted + arguments("dir1/dir2/dir3/key", "dir1/dir2/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", + KEY_COUNT * 2, 2, false), + // dir5, and all keys under dir3 and dir5 deleted + arguments("dir1/dir2/dir3/key", "dir1/dir2/dir5/key", "dir1/dir2/", "dir1/dir2/dir3", KEY_COUNT * 2, 1, true), + // dir2 dir3 dir4 dir5, and all keys under dir3 and dir5 deleted + arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT * 2, 4, false), + // dir4 dir5, and all keys under dir3 and dir5 deleted + arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "dir1/", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true), + // dir1 - dir5, and all keys deleted arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "", "dir1/dir2/dir3", KEY_COUNT * 2, 5, false), - arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true)); + // dir4 dir5, and all keys deleted + arguments("dir1/dir2/dir3/key", "dir1/dir4/dir5/key", "", "dir1/dir2/dir3", KEY_COUNT * 2, 2, true), + // dir4 dir5, and all keys under dir5 deleted + arguments("dir11/dir4/dir5/key", "dir1/dir2/dir3/key", "dir11/", "dir11/dir4/dir5", KEY_COUNT, 2, false), + // no dir, but all keys under dir11 deleted + arguments("dir11/dir4/dir5/key", "dir1/dir2/dir3/key", "dir11/", "dir11/dir4/dir5", KEY_COUNT, 0, true)); } @ParameterizedTest @@ -1506,7 +2466,7 @@ void testMultipleDirectoriesMatched(String keyPrefix1, String keyPrefix2, String LOG.info("expiry date {}", date.toInstant()); ZonedDateTime endDate = date.plus(SERVICE_INTERVAL, ChronoUnit.MILLIS); - GenericTestUtils.waitFor(() -> endDate.isBefore(ZonedDateTime.now(ZoneOffset.UTC)), WAIT_CHECK_INTERVAL, 10000); + GenericTestUtils.waitFor(() -> endDate.isBefore(ZonedDateTime.now(ZoneOffset.UTC)), WAIT_CHECK_INTERVAL, 5000); // rename a key under directory to change directory's Modification time if (updateDirModificationTime) { @@ -1555,7 +2515,7 @@ void testGetLifecycleServiceStatus() throws Exception { ZonedDateTime date = ZonedDateTime.now(ZoneOffset.UTC).plusSeconds(EXPIRE_SECONDS); KeyLifecycleService.setInjectors( Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl())); - createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, prefix, null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, FILE_SYSTEM_OPTIMIZED, "", null, date.toString(), true); Thread.sleep(SERVICE_INTERVAL + 100); // Verify service is running and processing the bucket @@ -1959,6 +2919,7 @@ public Stream parameters1() { @MethodSource("parameters1") void testKeyDeletedOrRenamed(BucketLayout bucketLayout, boolean deleted) throws IOException, InterruptedException, TimeoutException { + assumeTrue(stateSaveInternal != -1); final String volumeName = getTestName(); final String bucketName = uniqueObjectName("bucket"); GenericTestUtils.LogCapturer log = @@ -1967,12 +2928,13 @@ void testKeyDeletedOrRenamed(BucketLayout bucketLayout, boolean deleted) GenericTestUtils.LogCapturer requestLog = GenericTestUtils.LogCapturer.captureLogs( LoggerFactory.getLogger(OMKeysDeleteRequest.class)); - String prefix = "key"; + String keyPrefix = "key"; + String rulePrefix = bucketLayout == FILE_SYSTEM_OPTIMIZED ? "" : keyPrefix; long initialDeletedKeyCount = getDeletedKeyCount(); long initialKeyCount = getKeyCount(bucketLayout); // create keys List keyList = - createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, prefix, null); + createKeys(volumeName, bucketName, bucketLayout, KEY_COUNT, 1, keyPrefix, null); KeyLifecycleService.setInjectors( Arrays.asList(new FaultInjectorImpl(), new FaultInjectorImpl())); @@ -1980,13 +2942,14 @@ void testKeyDeletedOrRenamed(BucketLayout bucketLayout, boolean deleted) // create Lifecycle configuration ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); ZonedDateTime date = now.plusSeconds(EXPIRE_SECONDS); - createLifecyclePolicy(volumeName, bucketName, bucketLayout, prefix, null, date.toString(), true); + createLifecyclePolicy(volumeName, bucketName, bucketLayout, rulePrefix, null, date.toString(), true); Thread.sleep(SERVICE_INTERVAL); KeyLifecycleService.getInjector(0).resume(); GenericTestUtils.waitFor( () -> log.getOutput().contains(KEY_COUNT + " expired keys and 0 expired dirs found"), WAIT_CHECK_INTERVAL, 10000); + OmKeyArgs key = keyList.get(ThreadLocalRandom.current().nextInt(1, keyList.size())); // delete/rename another key before send deletion requests if (deleted) { From 627e6313ea28151822bef2548025bc8b0b160bda Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 23 Jun 2026 14:56:50 +0800 Subject: [PATCH 2/4] address comments --- .../content/design/lifecycle-task-resume.md | 10 +- .../apache/hadoop/ozone/om/OMConfigKeys.java | 1 - .../om/request/key/OMKeysDeleteRequest.java | 31 ++++-- .../om/response/key/OMKeysDeleteResponse.java | 4 + .../key/OMKeysDeleteResponseWithFSO.java | 3 +- .../ozone/om/service/KeyLifecycleService.java | 104 ++++-------------- .../om/service/TestKeyLifecycleService.java | 3 +- 7 files changed, 56 insertions(+), 100 deletions(-) diff --git a/hadoop-hdds/docs/content/design/lifecycle-task-resume.md b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md index 0898b19f813b..d800e21d4fe4 100644 --- a/hadoop-hdds/docs/content/design/lifecycle-task-resume.md +++ b/hadoop-hdds/docs/content/design/lifecycle-task-resume.md @@ -13,7 +13,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ +*/ # Design for Resumable Lifecycle Scans(HDDS-8342) @@ -46,21 +46,21 @@ message LifecycleScanState { ``` ### OM DB Schema Updates -Add a new table `lifecycleStateTable` to `OMMetadataManager` to store the scan states: -- **Table Name:** `lifecycleStateTable` +Add a new table `lifecycleScanStateTable` to `OMMetadataManager` to store the scan states: +- **Table Name:** `lifecycleScanStateTable` - **Key:** `bucketKey` (String, e.g., `/volumeName/bucketName`) - **Value:** `LifecycleScanState` ### When to Persist the Pointer Persisting the pointer for every key would overwhelm Ratis and RocksDB. We should checkpoint periodically: -1. **Piggybacking on Deletes:** Add an optional `LifecycleScanState` field to `DeleteKeysRequest`. When the OM state machine applies the deletion, it atomically updates the `lifecycleStateTable` with the new pointer. This guarantees exactly-once semantics for the scan pointer relative to deletions. +1. **Piggybacking on Deletes:** Add an optional `LifecycleScanState` field to `DeleteKeysRequest`. When the OM state machine applies the deletion, it atomically updates the `lifecycleScanStateTable` with the new pointer. This guarantees exactly-once semantics for the scan pointer relative to deletions. 2. **Move to trash**: Since there is no `RenameKeysRequest`, rename has be called multiple times for a batch of keys. We introduce a new OM request `SaveLifecycleScanStateRequest`. After a batch of keys are moved to trash, call `SaveLifecycleScanStateRequest` explicitly to persist the state. 3. **Periodic Standalone Checkpoints:** If no keys are expired (e.g., scanning millions of valid keys), we still need to save progress. The `LifecycleActionTask` will send this request periodically (e.g., every 100,000 keys iterated, or every 1 minute of execution time). 3. **End of Scan:** When the scan for a bucket finishes, a `SaveLifecycleScanStateRequest` is sent to mark state as completed by recording the completion time. ### How to Resume the Scan -When `KeyLifecycleService` schedules a `LifecycleActionTask` for a bucket, it first reads the `LifecycleScanState` from the `lifecycleStateTable`. +When `KeyLifecycleService` schedules a `LifecycleActionTask` for a bucket, it first reads the `LifecycleScanState` from the `lifecycleScanStateTable`. - **OBS/Legacy Resumption:** The iterator for `keyTable` is initialized to seek to `lastScannedKey` instead of the bucket prefix. diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 709f1f4d94e0..7d2d0e860bd9 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -199,7 +199,6 @@ public final class OMConfigKeys { "ozone.lifecycle.service.delete.cached.directory.max-count"; public static final long OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT = 1000000; - // Save task state for every 5m, or evaluated keys reaches 100k public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS = "ozone.lifecycle.service.state.save.interval.ms"; public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT = 5 * 60 * 1000; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java index e1d9a66ce3fd..528badf912aa 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysDeleteRequest.java @@ -40,6 +40,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; @@ -93,6 +94,25 @@ public OMKeysDeleteRequest(OMRequest omRequest, BucketLayout bucketLayout) { super(omRequest, bucketLayout); } + @Override + public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { + DeleteKeysRequest deleteKeysRequest = super.preExecute(ozoneManager).getDeleteKeysRequest(); + Objects.requireNonNull(deleteKeysRequest, "deleteKeysRequest == null"); + + if (deleteKeysRequest.getSourceType() == RequestSource.LIFECYCLE && deleteKeysRequest.hasScanState()) { + if (ozoneManager.getAclsEnabled()) { + UserGroupInformation ugi = createUGIForApi(); + if (!ozoneManager.isAdmin(ugi)) { + throw new OMException("Access denied for user " + ugi + ". " + + "Superuser privilege is required to save Lifecycle Service task state.", + OMException.ResultCodes.ACCESS_DENIED); + } + } + } + + return getOmRequest(); + } + @Override @SuppressWarnings("methodlength") public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, ExecutionContext context) { final long trxnLogIndex = context.getIndex(); @@ -164,17 +184,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut validateBucketAndVolume(omMetadataManager, volumeName, bucketName); String volumeOwner = getVolumeOwner(omMetadataManager, volumeName); - if (sourceType == RequestSource.LIFECYCLE && deleteKeyRequest.hasScanState()) { - if (ozoneManager.getAclsEnabled()) { - UserGroupInformation ugi = createUGIForApi(); - if (!ozoneManager.isAdmin(ugi)) { - throw new OMException("Access denied for user " + ugi + ". " - + "Superuser privilege is required to save Lifecycle Service task state.", - OMException.ResultCodes.ACCESS_DENIED); - } - } - } - for (indexFailed = 0; indexFailed < length; indexFailed++) { String keyName = deleteKeyArgs.getKeys(indexFailed); String objectKey = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java index a8eff37bf661..fd089005fcd8 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponse.java @@ -129,4 +129,8 @@ public OmBucketInfo getOmBucketInfo() { protected Map getOpenKeyInfoMap() { return openKeyInfoMap; } + + public OmLifecycleScanState getScanState() { + return scanState; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java index e955f45a980e..7bdcfa00b5b6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeysDeleteResponseWithFSO.java @@ -48,7 +48,6 @@ public class OMKeysDeleteResponseWithFSO extends OMKeysDeleteResponse { private List dirsList; private long volumeId; - private OmLifecycleScanState scanState; public OMKeysDeleteResponseWithFSO( @Nonnull OzoneManagerProtocolProtos.OMResponse omResponse, @@ -60,7 +59,6 @@ public OMKeysDeleteResponseWithFSO( super(omResponse, keyDeleteList, omBucketInfo, openKeyInfoMap, scanState); this.dirsList = dirDeleteList; this.volumeId = volId; - this.scanState = scanState; } @Override @@ -107,6 +105,7 @@ public void addToDBBatch(OMMetadataManager omMetadataManager, } } + OmLifecycleScanState scanState = getScanState(); if (scanState != null) { omMetadataManager.getLifecycleScanStateTable().putWithBatch( batchOperation, scanState.getBucketKey(), scanState); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index a14bc0ed0a05..a3df126fdb05 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -171,7 +171,7 @@ public KeyLifecycleService(OzoneManager ozoneManager, LOG.warn("Illegal value {} for Property {}. Set {} to {}", stateSaveIntervalMs, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT); - maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; + stateSaveIntervalMs = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT; } this.maxKeysProcessedPerState = conf.getLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT); @@ -181,6 +181,7 @@ public KeyLifecycleService(OzoneManager ozoneManager, OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT); maxKeysProcessedPerState = OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT; } + LOG.info("stateSaveIntervalMs = {}, maxKeysProcessedPerState = {}", stateSaveIntervalMs, maxKeysProcessedPerState); this.inFlight = new ConcurrentHashMap(); this.omMetadataManager = ozoneManager.getMetadataManager(); int limit = (int) conf.getStorageSize( @@ -629,40 +630,21 @@ private List getRuleUnion(long volumeId, OmBucketInfo return sortedConsolidatedRules; } - private boolean canSkipDir(OmDirectoryInfo currentDir, String currentDirTableKey, DirectoryList dirList) { - // currentDir null is bucket root - if (currentDir == null) { + private boolean canSkipDir(String currentDirPath, String lastScannedDirInState) { + if (lastScannedDirInState == null || currentDirPath.isEmpty()) { return false; } - - int count = dirList.getSubDirCount(); - // if currentDir is equal to lastScannedDir - if (currentDir.getObjectID() == dirList.getSubDirList().get(count - 1).getObjectID()) { - return false; - } - - // if currentDir is parent of lastScannedDir - long currentObjID = currentDir.getObjectID(); - for (int i = 0; i < count; i++) { - OmDirectoryInfo dir = dirList.getSubDirList().get(i); - if (dir.getObjectID() == currentObjID) { - return false; + String[] cur = currentDirPath.split(OM_KEY_PREFIX); + String[] last = lastScannedDirInState.split(OM_KEY_PREFIX); + int n = Math.min(cur.length, last.length); + for (int i = 0; i < n; i++) { + int cmp = cur[i].compareTo(last[i]); + if (cmp != 0) { + // current name > last name -> skip + return cmp > 0; } } - - // if currentDir and lastScannedDir has same parent - do { - long parentID = currentDir.getParentObjectID(); - for (int i = 0; i < count; i++) { - OmDirectoryInfo dir = dirList.getSubDirList().get(i); - if (dir.getParentObjectID() == parentID) { - if (dirList.getSubDirKeyList().get(i).compareTo(currentDirTableKey) < 0) { - return true; - } - } - } - return false; - } while (true); + return false; } @SuppressWarnings({"checkstyle:parameternumber", "checkstyle:MethodLength"}) @@ -676,19 +658,6 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table String lastScannedDirInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDir(); String lastScannedDirKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedDirKey(); String lastScannedKeyInState = scanStateBuilder == null ? null : scanStateBuilder.getLastScannedKey(); - DirectoryList lastScannedDirList = null; - String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); - if (lastScannedDirInState != null && lastScannedDirKeyInState != null) { - // find all parents of lastScannedSubDir - try { - lastScannedDirList = getDirList(volumeObjId, bucket, lastScannedDirInState, bucketKey); - } catch (IOException e) { - // Saved lastScannedDir in state could be deleted or renamed after it's saved. - // Fallback to no state saved status. - LOG.info("Failed to get DirList for lastScannedDirInState {}", lastScannedDirInState, e); - lastScannedDirInState = null; - } - } try { if (dir != null) { stack.push(new PendingEvaluateDirectory(dir, dirKey, directoryPath, null)); @@ -748,8 +717,7 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table * - dir3/dir8, lastScannedDir, partially scanned, * - dir3/dir9, same the same parentID as lastScannedDir, and name order > lastScannedDir, scanned, skip */ - if (lastScannedDirList != null && - canSkipDir(currentDir, currentDirTableKey, lastScannedDirList)) { + if (canSkipDir(currentDirPath, lastScannedDirInState)) { LOG.info("Skip {} in LifecycleActionTask for bucket {}. ", currentDirPath, bucketName); continue; } @@ -885,33 +853,12 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table try (TableIterator> keyTblItr = keyTable.iterator(prefix)) { + boolean seekPerformed = false; if (lastScannedDirKeyInState != null && lastScannedDirKeyInState.compareTo(currentDirTableKey) == 0 && lastScannedKeyInState != null && lastScannedKeyInState.startsWith(prefix)) { LOG.info("Seek to key {} under directory {}", scanStateBuilder.getLastScannedKey(), lastScannedDirInState); keyTblItr.seek(scanStateBuilder.getLastScannedKey()); - if (keyTblItr.hasNext()) { - Table.KeyValue first = keyTblItr.next(); - if (!first.getKey().equals(scanStateBuilder.getLastScannedKey())) { - OmKeyInfo key = first.getValue(); - String keyPath = currentDirPath.isEmpty() ? key.getKeyName() : - currentDirPath + OM_KEY_PREFIX + key.getKeyName(); - if (!deletedKeySetInCache.remove(first.getKey()) && !keySetInCache.remove(first.getKey())) { - numKeyIterated++; - numKeysUnderDir++; - for (OmLCRule rule : ruleList) { - if (key.getParentObjectID() == currentDirObjID && rule.match(key, keyPath)) { - if (keyList.isFull()) { - handleAndClearFullList(bucket, keyList, false, scanStateBuilder, false); - } - keyList.add(keyPath, key.getReplicatedSize(), key.getUpdateID()); - numKeysExpired++; - break; - } - } - lastScannedKey = first.getKey(); - } - } - } + seekPerformed = true; } while (keyTblItr.hasNext()) { @@ -923,6 +870,9 @@ private void evaluateKeyAndDirTable(OmBucketInfo bucket, long volumeObjId, Table OmKeyInfo key = keyValue.getValue(); String keyPath = currentDirPath.isEmpty() ? key.getKeyName() : currentDirPath + OM_KEY_PREFIX + key.getKeyName(); + if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) { + continue; + } if (deletedKeySetInCache.remove(keyValue.getKey()) || keySetInCache.remove(keyValue.getKey())) { continue; } @@ -1066,19 +1016,10 @@ private void evaluateBucket(OmBucketInfo bucketInfo, try (TableIterator> keyTblItr = keyTable.iterator(bucketPrefix)) { + boolean seekPerformed = false; if (scanStateBuilder != null && scanStateBuilder.getLastScannedKey() != null) { keyTblItr.seek(scanStateBuilder.getLastScannedKey()); - // Skip the exact match since it was already processed - if (keyTblItr.hasNext()) { - Table.KeyValue first = keyTblItr.next(); - if (!first.getKey().equals(scanStateBuilder.getLastScannedKey())) { - // We seeked past it, so we need to process this one. - // We can't easily "push back" in TableIterator, so we handle it here. - processKey(bucketInfo, first.getValue(), ruleList, expiredKeyList, scanStateBuilder); - numKeyIterated++; - lastScannedKey = first.getKey(); - } - } + seekPerformed = true; } while (keyTblItr.hasNext()) { @@ -1091,6 +1032,9 @@ private void evaluateBucket(OmBucketInfo bucketInfo, flushAndSaveState(bucketInfo, expiredKeyList, null, scanStateBuilder); } Table.KeyValue keyValue = keyTblItr.next(); + if (seekPerformed && keyValue.getKey().equals(scanStateBuilder.getLastScannedKey())) { + continue; + } processKey(bucketInfo, keyValue.getValue(), ruleList, expiredKeyList, scanStateBuilder); numKeyIterated++; lastScannedKey = keyValue.getKey(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java index 6e741d404f23..6b8f2e6e12ed 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java @@ -219,6 +219,7 @@ private void createConfig(File testDir) { conf.setLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS, stateSaveInternal); conf.setLong(OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED, maxKeysProcessedPerState); OmLCExpiration.setTest(true); + KeyLifecycleService.setTest(true); } private void createSubject() throws Exception { @@ -1083,7 +1084,7 @@ void testBucketRootScannedDirResume(BucketLayout layout) throws Exception { // So key1 and key2 are skipped, key3 is deleted. int expectedDeleted = 1; GenericTestUtils.waitFor(() -> - (getDeletedKeyCount() - initialDeletedKeyCount) >= expectedDeleted, WAIT_CHECK_INTERVAL, 10000); + (getDeletedKeyCount() - initialDeletedKeyCount) == expectedDeleted, WAIT_CHECK_INTERVAL, 10000); assertEquals(2, getKeyCount(layout) - initialKeyCount); GenericTestUtils.waitFor(() -> From 7c09d3c58a72d627c127bc2fd0379a42121d5e61 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 13 Jul 2026 10:06:54 +0800 Subject: [PATCH 3/4] address comments --- .../request/lifecycle/OMLifecycleSaveScanStateRequest.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java index 94c4d3faa1b9..94ef71f0b01c 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java @@ -17,6 +17,7 @@ package org.apache.hadoop.ozone.om.request.lifecycle; +import java.io.IOException; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OzoneManager; @@ -42,7 +43,8 @@ public OMLifecycleSaveScanStateRequest(OMRequest omRequest) { } @Override - public OMRequest preExecute(OzoneManager ozoneManager) throws OMException { + public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { + OMRequest omRequest = super.preExecute(ozoneManager); if (ozoneManager.getAclsEnabled()) { UserGroupInformation ugi = createUGIForApi(); if (!ozoneManager.isAdmin(ugi)) { @@ -51,7 +53,7 @@ public OMRequest preExecute(OzoneManager ozoneManager) throws OMException { OMException.ResultCodes.ACCESS_DENIED); } } - return getOmRequest(); + return omRequest; } @Override From 735bdf7bcf42d3f1be07eec148e16710ab4e33b8 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Mon, 13 Jul 2026 13:32:52 +0800 Subject: [PATCH 4/4] fix UT --- .../TestOMLifecycleSaveScanStateRequest.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java index 96c0508a04b3..8c5a806ac942 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java @@ -17,6 +17,8 @@ package org.apache.hadoop.ozone.om.request.lifecycle; +import static org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager.maxLayoutVersion; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -32,6 +34,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmLifecycleScanState; import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleScanState; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; @@ -47,6 +50,9 @@ public class TestOMLifecycleSaveScanStateRequest { @Test public void testPreExecuteAdminCheck() throws Exception { OzoneManager ozoneManager = mock(OzoneManager.class); + OMLayoutVersionManager versionManager = mock(OMLayoutVersionManager.class); + when(versionManager.getMetadataLayoutVersion()).thenReturn(maxLayoutVersion()); + when(ozoneManager.getVersionManager()).thenReturn(versionManager); // Test when ACLs are enabled but user is not admin when(ozoneManager.getAclsEnabled()).thenReturn(true); @@ -72,14 +78,12 @@ public void testPreExecuteAdminCheck() throws Exception { // Test when user is admin when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(true); - OMRequest preExecuted = request.preExecute(ozoneManager); - assertEquals(omRequest, preExecuted); - + assertDoesNotThrow(() -> request.preExecute(ozoneManager)); + // Test when ACLs are disabled when(ozoneManager.getAclsEnabled()).thenReturn(false); when(ozoneManager.isAdmin(any(UserGroupInformation.class))).thenReturn(false); - preExecuted = request.preExecute(ozoneManager); - assertEquals(omRequest, preExecuted); + assertDoesNotThrow(() -> request.preExecute(ozoneManager)); } @Test @@ -87,7 +91,6 @@ public void testValidateAndUpdateCache() throws Exception { OzoneManager ozoneManager = mock(OzoneManager.class); OMMetadataManager omMetadataManager = mock(OMMetadataManager.class); when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); - Table table = mock(Table.class); when(omMetadataManager.getLifecycleScanStateTable()).thenReturn(table); @@ -108,10 +111,6 @@ public void testValidateAndUpdateCache() throws Exception { .build(); OMLifecycleSaveScanStateRequest request = new OMLifecycleSaveScanStateRequest(omRequest); - - OMRequest preExecuted = request.preExecute(ozoneManager); - assertEquals(omRequest, preExecuted); - OMClientResponse response = request.validateAndUpdateCache(ozoneManager, 100L); assertNotNull(response); assertEquals(OzoneManagerProtocolProtos.Status.OK, response.getOMResponse().getStatus());