HDDS-15447. Persist bucket scanned key pointer periodically

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -5140,4 +5140,19 @@
       lifecycle rules.
     </description>
   </property>
+  <property>
+    <name>ozone.lifecycle.service.state.save.interval.ms</name>
+    <value>300000</value>
+    <tag>OZONE</tag>
+    <description>The interval of bucket scan task saves its pointer to DB. Default is 5 mins.</description>
+  </property>
+  <property>
+    <name>ozone.lifecycle.service.state.save.keys.processed</name>
+    <value>100000</value>
+    <tag>OZONE</tag>
+    <description>Bucket scan task will save its pointer to DB by default every 100000 keys are scanned. Bucket scan
+      pointer save will happen when either ozone.lifecycle.service.state.save.interval.ms or
+      ozone.lifecycle.service.state.save.keys.processed is satisfied.
+    </description>
+  </property>
 </configuration>

hadoop-hdds/docs/content/design/lifecycle-task-resume.md

@@ -0,0 +1,80 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+# Design for Resumable Lifecycle Scans(HDDS-8342)
+
+## Problem Statement:
+
+The `HDDS-8342` branch introduces the `KeyLifecycleService`, a background service running on the Ozone Manager (OM) Leader to enforce bucket lifecycle rules (expiration, moving to trash, and aborting incomplete multipart uploads).
+The entire bucket is scanned in a single `call()` execution. If the OM restarts, crashes, or a leader transfer occurs, the scan state is lost. The new leader must restart the scan from the beginning. 
+For buckets with billions of keys, the scan may never complete if leader transfers happen periodically.
+
+## Design: Persisting Bucket Scan Pointers
+
+To solve the resumability issues, we need to persist the scan progress (the "pointer") to the OM DB. This ensures that a new OM leader can resume from where the previous leader left off.
+
+### 2.1 Data Structure for the Scan Pointer
+
+Define a new Protobuf message `LifecycleScanState` to capture the exact position of the scan.
+
+```protobuf
+message LifecycleScanState {
+    optional string bucketKey = 1;  // e.g., /volume/bucket
+    optional uint64 bucketObjID = 2;  // bucket's object ID, in case the bucket is deleted and recreated with same name
+    optional uint64 lifecycleConfigurationUpdateID = 3;  // lifecycle configuration update ID, in case the bucket is updated with new rules
+    optional uint64 scanStartTime = 4;  // Epoch time when this full scan started
+    optional uint64 scanEndTime = 5;  // Epoch time when this full scan is completed
+    optional string lastScannedKey = 6;  // the last scanned key in the bucket(for both OBS and FSO)
+    optional string lastScannedDir = 7;  // the last scanned dir path, e.g /dir1/dir2/dir3
+    optional string lastScannedDirKey = 8; // the last scanned dir key in directoryTable, e.g /0/1/3/dir3
+    optional string lastScannedMpuKey = 9;
+}
+```
+
+### OM DB Schema Updates
+Add a new table `lifecycleScanStateTable` to `OMMetadataManager` to store the scan states:
+- **Table Name:** `lifecycleScanStateTable`
+- **Key:** `bucketKey` (String, e.g., `/volumeName/bucketName`)
+- **Value:** `LifecycleScanState`
+
+### When to Persist the Pointer
+Persisting the pointer for every key would overwhelm Ratis and RocksDB. We should checkpoint periodically:
+
+1. **Piggybacking on Deletes:** Add an optional `LifecycleScanState` field to `DeleteKeysRequest`. When the OM state machine applies the deletion, it atomically updates the `lifecycleScanStateTable` with the new pointer. This guarantees exactly-once semantics for the scan pointer relative to deletions.
+2. **Move to trash**: Since there is no `RenameKeysRequest`, rename has be called multiple times for a batch of keys. We introduce a new OM request `SaveLifecycleScanStateRequest`. After a batch of keys are moved to trash, call `SaveLifecycleScanStateRequest` explicitly to persist the state.
+3. **Periodic Standalone Checkpoints:** If no keys are expired (e.g., scanning millions of valid keys), we still need to save progress. The `LifecycleActionTask` will send this request periodically (e.g., every 100,000 keys iterated, or every 1 minute of execution time).
+3. **End of Scan:** When the scan for a bucket finishes, a `SaveLifecycleScanStateRequest` is sent to mark state as completed by recording the completion time.
+
+### How to Resume the Scan
+When `KeyLifecycleService` schedules a `LifecycleActionTask` for a bucket, it first reads the `LifecycleScanState` from the `lifecycleScanStateTable`.
+
+- **OBS/Legacy Resumption:** 
+  The iterator for `keyTable` is initialized to seek to `lastScannedKey` instead of the bucket prefix.
+  ```java
+  TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>> keyTblItr = keyTable.iterator(bucketPrefix);
+  if (state.getLastScannedKey() != null) {
+      keyTblItr.seek(state.getLastScannedKey());
+      // skip the exact match since it was already processed
+  }
+  ```
+
+- **FSO Resumption:**
+  Since FSO bucket is iterated via a Depth-First Search (DFS) way, any directory that is after the `lastScannedDir` in the traversal path can be skipped. 
+
+- **MPU Resumption:**
+  // TODO: implement MPU resumption
+  The `multipartInfoTable` iterator seeks to `lastScannedMpuKey` and continues.

hadoop-hdds/docs/content/feature/Lifecycle.md

@@ -132,19 +132,22 @@ For FILE_SYSTEM_OPTIMIZED (FSO) buckets, the Prefix must be a normalized and val
 - Cannot start with `/`. FSO bucket prefixes are relative to the bucket root and do not need a leading slash.
 - Cannot contain consecutive slashes `//`.
 - Path components cannot contain `.` (current directory), `..` (parent directory), or `:`.
+- Must end with "/", or "" for root.
 
 The following table shows examples of valid and invalid prefixes:
 
 | Prefix | Valid for FSO Bucket | Reason |
-|--------|----------------------|--------|
+|----|----------------------|--|
 | `logs/` | Valid | Normalized directory prefix |
 | `data/2024/` | Valid | Multi-level directory prefix |
-| `archive` | Valid | Simple prefix without slash |
+| `archive` | Invalid | Without tailing slash |
 | `/logs/` | Invalid | Cannot start with `/`, use `logs/` instead |
 | `data//backup/` | Invalid | Contains consecutive slashes `//`, use `data/backup/` instead |
 | `data/../secret/` | Invalid | Contains `..`, parent directory references are not allowed |
 | `data/./logs/` | Invalid | Contains `.`, current directory references are not allowed |
 | `.Trash/` | Invalid | Cannot point to trash directories |
+| `` | Valid | Point to Bucket's root directory |
+| `/` | Invalid | It doesn't point to Bucket's root directory. Use "" instead |
 
 ## S3 Gateway API
 
@@ -354,9 +357,9 @@ ozone admin om lifecycle resume [-id=<omServiceId>] [-host=<omHost>]
 In an OM HA deployment, the lifecycle service only runs on the leader OM. When a Transfer Leader operation is performed:
 
 1. The lifecycle evaluation tasks running on the old leader will be interrupted.
-2. After the new leader is elected, the lifecycle service restarts from the beginning. Previously evaluated buckets are not skipped, and the task starts over from the first bucket.
+2. After the new leader is elected, the lifecycle service will skip the previously evaluated buckets/bucket contents, starting from the interrupted bucket.
 
-Therefore, in scenarios with frequent leader transfers, it is recommended to monitor the actual execution of the lifecycle service to ensure expired objects are cleaned up in a timely manner.
+In scenarios with frequent leader transfers, it is recommended to monitor the actual execution of the lifecycle service to ensure expired objects are cleaned up in a timely manner.
 
 ### Impact of Mass Key Expiration on Metadata Performance
 

hadoop-hdds/docs/content/feature/Lifecycle.zh.md

@@ -131,19 +131,22 @@ FSO Bucket 的额外校验规则：
 - 不能以 `/` 开头。FSO Bucket 的 Prefix 是相对于 Bucket 根目录的路径，无需前导斜杠。
 - 不能包含连续的斜杠 `//`。
 - 路径组件中不能包含 `.`（当前目录）、`..`（父目录）或 `:`。
+- 路径必须以 `/` 结尾， 或者 “” 代表root。
 
 以下是合法与不合法 Prefix 的对照示例：
 
 | Prefix | FSO Bucket 是否合法 | 原因 |
-|--------|---------------------|------|
+|----|--------------------|-----|
 | `logs/` | 合法 | 规范化的目录前缀 |
 | `data/2024/` | 合法 | 多级目录前缀 |
-| `archive` | 合法 | 无斜杠的简单前缀 |
+| `archive` | 不合法 | 无斜杠结尾 |
 | `/logs/` | 不合法 | 不能以 `/` 开头，应使用 `logs/` |
 | `data//backup/` | 不合法 | 包含连续斜杠 `//`，应使用 `data/backup/` |
 | `data/../secret/` | 不合法 | 包含 `..`，不允许使用父目录引用 |
 | `data/./logs/` | 不合法 | 包含 `.`，不允许使用当前目录引用 |
 | `.Trash/` | 不合法 | 不能指向回收站目录 |
+| `` | 合法 | 指向Bucket 根目录 |
+| `/` | 不合法 | 它不指向 Bucket 根目录。正确代表根目录的前缀是 “” |
 
 ## S3 Gateway API
 
@@ -353,9 +356,9 @@ ozone admin om lifecycle resume [-id=<omServiceId>] [-host=<omHost>]
 在 OM HA 部署中，生命周期服务仅在 Leader OM 上运行。当执行 Transfer Leader 操作时：
 
 1. 旧 Leader 上正在运行的生命周期评估任务会被中断。
-2. 新 Leader 当选后，会重新从头启动生命周期服务，已经评估过的 Bucket 不会被跳过，任务将从第一个 Bucket 重新开始。
+2. 新 Leader 当选后，会重新从头启动生命周期服务，已经评估过的 Bucket 会被跳过，任务将从中断的Bucket 重新开始。
 
-因此，在频繁进行 Leader 切换的场景下，建议关注生命周期服务的实际执行情况，确保过期对象能被及时清理。
+在频繁进行 Leader 切换的场景下，建议关注生命周期服务的实际执行情况，确保过期对象能被及时清理。
 
 ### 大批量 Key 过期对元数据性能的影响
 

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java

@@ -327,6 +327,7 @@ public static boolean isReadOnly(OMRequest omRequest) {
     case SetLifecycleConfiguration:
     case DeleteLifecycleConfiguration:
     case SetLifecycleServiceStatus:
+    case SaveLifecycleScanState:
     case UnknownCommand:
       return false;
     case EchoRPC:
@@ -462,6 +463,7 @@ public static boolean shouldSendToFollower(OMRequest omRequest) {
     case SetLifecycleConfiguration:
     case DeleteLifecycleConfiguration:
     case SetLifecycleServiceStatus:
+    case SaveLifecycleScanState:
     case UnknownCommand:
       return false;
     case EchoRPC:

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java

@@ -199,6 +199,13 @@ public final class OMConfigKeys {
       "ozone.lifecycle.service.delete.cached.directory.max-count";
   public static final long OZONE_KEY_LIFECYCLE_SERVICE_DELETE_CACHED_DIRECTORY_MAX_COUNT_DEFAULT = 1000000;
 
+  public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS =
+      "ozone.lifecycle.service.state.save.interval.ms";
+  public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_INTERVAL_MS_DEFAULT = 5 * 60 * 1000;
+  public static final String OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED =
+      "ozone.lifecycle.service.state.save.keys.processed";
+  public static final long OZONE_KEY_LIFECYCLE_SERVICE_STATE_SAVE_KEYS_PROCESSED_DEFAULT = 100000;
+
   public static final String OZONE_KEY_LIFECYCLE_SERVICE_MOVE_TO_TRASH_ENABLED =
       "ozone.lifecycle.service.move.to.trash.enabled";
   public static final boolean

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java

@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.ozone.om.helpers;
 
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
 import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateAndNormalizePrefix;
 import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validatePrefixLength;
 import static org.apache.hadoop.ozone.om.helpers.OmLifecycleUtils.validateTrashPrefix;
@@ -28,7 +29,6 @@
 import net.jcip.annotations.Immutable;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction;
 import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule;
@@ -61,7 +61,7 @@ private OmLCRule() {
   private OmLCRule(Builder builder) {
     this.prefix = builder.prefix;
     if (this.prefix != null) {
-      this.directoryStylePrefix = this.prefix.contains(OzoneConsts.OM_KEY_PREFIX);
+      this.directoryStylePrefix = this.prefix.contains(OM_KEY_PREFIX);
     } else {
       this.directoryStylePrefix = false;
     }
@@ -196,6 +196,14 @@ public void valid(BucketLayout bucketLayout, Long creationTime) throws OMExcepti
             OMException.ResultCodes.INVALID_REQUEST);
       }
       action.valid(creationTime);
+
+      if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
+        if (getEffectivePrefix() != null && !getEffectivePrefix().isEmpty() &&
+            !getEffectivePrefix().endsWith(OM_KEY_PREFIX)) {
+          throw new OMException("FILE_SYSTEM_OPTIMIZED bucket prefix must end with '/'.",
+              OMException.ResultCodes.INVALID_REQUEST);
+        }
+      }
     }
 
     if (prefix != null && filter != null) {
@@ -323,7 +331,6 @@ public static OmLCRule getFromProtobuf(LifecycleRule lifecycleRule, BucketLayout
     if (lifecycleRule.hasFilter()) {
       builder.setFilter(OmLCFilter.getFromProtobuf(lifecycleRule.getFilter(), layout));
     }
-
     return builder.build();
   }