From a17119b3990b20288f041483dd5cb6d8616c1f0f Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Sat, 25 Jul 2026 10:17:26 +0530 Subject: [PATCH 1/7] HDDS-15935. Extract ExportFileManager and document container export directory layout --- .../container/export/ExportFileManager.java | 179 ++++++++++++++++++ .../scm/container/export/ExportScope.java | 74 ++++++++ .../scm/container/export/package-info.java | 21 ++ .../export/TestExportFileManager.java | 97 ++++++++++ 4 files changed, 371 insertions(+) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/package-info.java create mode 100644 hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java new file mode 100644 index 000000000000..c194ebfa569d --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.export; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages on-disk paths and artifacts for container ID export jobs. + * Layout under the export directory ({@code {exportDirectory}}, typically {@code {scm.db.dirs}/exports}): + *

+ * {exportDirectory}/ + * {jobId}.in-progress // marker while a job is running + * container-ids-{scope}-{timestamp}-{jobId}.tar // completed export archive + * export-{jobId}/ // per-job workspace (removed on success) + * work/ + * container-ids-{scope}-{timestamp}-part001.txt + * ... + *

+ * Shard text files are written under {@code export-{jobId}/work/}, appended into the TAR at + * {@code {exportDirectory}}, then the manager deletes the workspace. The manager clears the + * {@code .in-progress} marker only after the TAR closes successfully. On startup, the manager + * removes orphaned markers, workspaces, and partial TAR files for the same job id together. + */ +final class ExportFileManager { + + private static final Logger LOG = LoggerFactory.getLogger(ExportFileManager.class); + static final String IN_PROGRESS_MARKER_SUFFIX = ".in-progress"; + static final String EXPORT_JOB_DIR_PREFIX = "export-"; + private final String exportDirectory; + + ExportFileManager(String exportDirectory) { + this.exportDirectory = Objects.requireNonNull(exportDirectory, "exportDirectory == null"); + } + + String getExportDirectory() { + return exportDirectory; + } + + void start() throws IOException { + Files.createDirectories(Paths.get(exportDirectory)); + cleanupOrphanedExportArtifacts(); + } + + String resolveTarPath(ExportScope scope, String fileTimestamp, String jobId) { + String tarFileName = String.format("container-ids-%s-%s-%s.tar", scope.getValue(), fileTimestamp, jobId); + return exportDirectory + File.separator + tarFileName; + } + + void markExportInProgress(String jobId) throws IOException { + Files.createFile(inProgressMarkerFile(jobId).toPath()); + } + + void clearExportInProgress(String jobId) { + FileUtils.deleteQuietly(inProgressMarkerFile(jobId)); + } + + void deleteExportTar(String tarPath) { + if (tarPath == null) { + return; + } + File tar = new File(tarPath); + if (tar.isFile() && FileUtils.deleteQuietly(tar)) { + LOG.debug("Removed container export TAR: {}", tar.getName()); + } + } + + void cleanupFailedArtifacts(Path jobDir, File tarFile, String jobId) { + if (jobDir != null) { + FileUtils.deleteQuietly(jobDir.toFile()); + } + if (tarFile != null) { + FileUtils.deleteQuietly(tarFile); + } + clearExportInProgress(jobId); + } + + private void cleanupOrphanedExportArtifacts() { + File exportDir = new File(exportDirectory); + File[] children = exportDir.listFiles(); + if (children == null) { + return; + } + for (File child : children) { + if (child.isFile() && child.getName().endsWith(IN_PROGRESS_MARKER_SUFFIX)) { + String jobId = child.getName().substring( + 0, child.getName().length() - IN_PROGRESS_MARKER_SUFFIX.length()); + if (isUuidDirectoryName(jobId)) { + removeIncompleteExportArtifacts(jobId); + } + } + } + for (File child : children) { + if (child.isDirectory()) { + String jobId = jobIdFromExportDirName(child.getName()); + if (jobId == null) { + continue; + } + if (inProgressMarkerFile(jobId).exists()) { + removeIncompleteExportArtifacts(jobId); + } else { + FileUtils.deleteQuietly(child); + } + } + } + } + + private void removeIncompleteExportArtifacts(String jobId) { + LOG.info("Removing incomplete container export artifacts for job {}", jobId); + FileUtils.deleteQuietly(inProgressMarkerFile(jobId)); + File tar = findTarForJobId(jobId); + if (tar != null) { + FileUtils.deleteQuietly(tar); + LOG.info("Removed incomplete container export TAR for job {}: {}", jobId, tar.getName()); + } + File jobWorkDir = new File(exportDirectory, exportJobDirName(jobId)); + if (jobWorkDir.isDirectory()) { + FileUtils.deleteQuietly(jobWorkDir); + LOG.info("Removed orphaned container export work directory: {}", jobWorkDir.getAbsolutePath()); + } + } + + private File findTarForJobId(String jobId) { + File exportDir = new File(exportDirectory); + File[] matches = exportDir.listFiles( + (dir, fileName) -> fileName.endsWith("-" + jobId + ".tar")); + if (matches == null || matches.length == 0) { + return null; + } + return matches[0]; + } + + private File inProgressMarkerFile(String jobId) { + return new File(exportDirectory, jobId + IN_PROGRESS_MARKER_SUFFIX); + } + + static String exportJobDirName(String jobId) { + return EXPORT_JOB_DIR_PREFIX + jobId; + } + + private static String jobIdFromExportDirName(String dirName) { + if (!dirName.startsWith(EXPORT_JOB_DIR_PREFIX)) { + return null; + } + String jobId = dirName.substring(EXPORT_JOB_DIR_PREFIX.length()); + return isUuidDirectoryName(jobId) ? jobId : null; + } + + private static boolean isUuidDirectoryName(String directoryName) { + try { + return directoryName.equals(UUID.fromString(directoryName).toString()); + } catch (IllegalArgumentException e) { + return false; + } + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java new file mode 100644 index 000000000000..c88cf886cc3f --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.export; + +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; +import org.apache.hadoop.hdds.scm.container.ContainerHealthState; + +/** + * Container listing filters for an export job. + * An export job filters containers by {@link ContainerHealthState}, {@link LifeCycleState} or both. + * Example TAR name: + * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z-{jobId}.tar} + */ +public final class ExportScope { + + private final LifeCycleState lifeCycleState; + private final ContainerHealthState healthState; + private final String value; + + private ExportScope(LifeCycleState lifeCycleState, ContainerHealthState healthState, String value) { + this.lifeCycleState = lifeCycleState; + this.healthState = healthState; + this.value = value; + } + + public static ExportScope of(LifeCycleState lifeCycleState, ContainerHealthState healthState) { + StringBuilder sb = new StringBuilder(); + if (healthState != null) { + sb.append("health-").append(healthState.name()); + } + if (lifeCycleState != null) { + if (sb.length() > 0) { + sb.append('_'); + } + sb.append("lifecycle-").append(lifeCycleState.name()); + } + return new ExportScope(lifeCycleState, healthState, sb.toString()); + } + + public LifeCycleState getLifeCycleState() { + return lifeCycleState; + } + + public ContainerHealthState getHealthState() { + return healthState; + } + + /** + * Stable filter name segment used in export TAR and shard file names. + */ + public String getValue() { + return value; + } + + @Override + public String toString() { + return value; + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/package-info.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/package-info.java new file mode 100644 index 000000000000..103c9519fcab --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This package contains classes related to container export. + */ +package org.apache.hadoop.hdds.scm.container.export; diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java new file mode 100644 index 000000000000..248e6f0e36d8 --- /dev/null +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.export; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.apache.hadoop.hdds.scm.container.ContainerHealthState; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** + * Tests for {@link ExportFileManager}. + */ +public class TestExportFileManager { + + @TempDir + private File tempDir; + + private ExportFileManager fileManager; + + @BeforeEach + public void setup() throws Exception { + fileManager = new ExportFileManager(tempDir.getAbsolutePath()); + fileManager.start(); + } + + @Test + public void testResolveTarPath() { + String jobId = UUID.randomUUID().toString(); + ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); + String tarPath = fileManager.resolveTarPath(scope, "20260101T120000Z", jobId); + assertTrue(tarPath.endsWith("container-ids-health-MISSING-20260101T120000Z-" + jobId + ".tar")); + } + + @Test + public void testOrphanWorkDirRemovedOnStartup() throws Exception { + String jobId = UUID.randomUUID().toString(); + Path orphan = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)).resolve("work"); + Files.createDirectories(orphan); + + fileManager.start(); + + assertFalse(Files.exists(orphan)); + } + + @Test + public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { + String jobId = UUID.randomUUID().toString(); + Path jobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)).resolve("work"); + Files.createDirectories(jobDir); + File partialTar = new File(tempDir, "container-ids-health-MISSING-20260101T000000Z-" + jobId + ".tar"); + assertTrue(partialTar.createNewFile()); + File inProgress = new File(tempDir, jobId + ExportFileManager.IN_PROGRESS_MARKER_SUFFIX); + assertTrue(inProgress.createNewFile()); + + fileManager.start(); + + assertFalse(Files.exists(jobDir)); + assertFalse(partialTar.exists()); + assertFalse(inProgress.exists()); + } + + @Test + public void testOrphanWorkDirWithoutMarkerDoesNotDeleteCompletedTar() throws Exception { + String jobId = UUID.randomUUID().toString(); + File completedTar = new File(tempDir, "container-ids-health-MISSING-20260101T000000Z-" + jobId + ".tar"); + assertTrue(completedTar.createNewFile()); + Path orphanWorkDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); + Files.createDirectories(orphanWorkDir.resolve("work")); + + fileManager.start(); + + assertTrue(completedTar.exists()); + assertFalse(Files.exists(orphanWorkDir)); + } +} From 0e6ce9b252f7003451bf925a759d1478824689a4 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Mon, 27 Jul 2026 10:55:58 +0530 Subject: [PATCH 2/7] Use gz and update javadoc --- .../container/export/ExportFileManager.java | 71 ++++++++++++------- .../scm/container/export/ExportScope.java | 2 +- .../export/TestExportFileManager.java | 27 +++---- 3 files changed, 63 insertions(+), 37 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java index c194ebfa569d..bb30a0ef2a51 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java @@ -30,26 +30,48 @@ /** * Manages on-disk paths and artifacts for container ID export jobs. - * Layout under the export directory ({@code {exportDirectory}}, typically {@code {scm.db.dirs}/exports}): - *

+ * + *

The export directory ({@code exportDirectory}, typically {@code {scm.db.dirs}/exports}) + * uses the layout below. The manager gzip-compresses the archive ({@code .tar.gz}) so operators + * can stream entries with {@code zcat} + * + * 

  * {exportDirectory}/
- *   {jobId}.in-progress                             // marker while a job is running
- *   container-ids-{scope}-{timestamp}-{jobId}.tar   // completed export archive
- *   export-{jobId}/                                 // per-job workspace (removed on success)
- *     work/
- *       container-ids-{scope}-{timestamp}-part001.txt
- *       ...
- * 

- * Shard text files are written under {@code export-{jobId}/work/}, appended into the TAR at
- * {@code {exportDirectory}}, then the manager deletes the workspace. The manager clears the
- * {@code .in-progress} marker only after the TAR closes successfully. On startup, the manager
- * removes orphaned markers, workspaces, and partial TAR files for the same job id together.
+ * ├── {jobId}.in-progress
+ * ├── container-ids-{scope}-{timestamp}-{jobId}.tar.gz
+ * └── export_{jobId}/
+ *     ├── container-ids-{scope}-{timestamp}-part001.txt
+ *     └── ...
+ *
+ * + *

{@code export_{jobId}/} holds shard text files while the job appends them into the archive. + * + *

When {@code export_{jobId}/} is deleted: the export manager deletes it after the + * archive closes successfully, or during {@link #cleanupFailedArtifacts} on failure or cancel. + * On startup, {@link #start()} deletes a leftover {@code export_{jobId}/} when no in-progress + * marker remains. If the marker still exists, {@link #start()} deletes {@code export_{jobId}/} + * together with the marker and any partial archive for that job id. + * + *

When {@code .tar.gz} is deleted: {@link #cleanupFailedArtifacts} deletes partial + * archives for failed or cancelled jobs. {@link #start()} deletes partial archives for jobs that + * still have an in-progress marker. Completed archives remain on disk until the export manager + * evicts the job from memory ({@code maxTerminalJobs} in {@code ContainerExportManager}) or an + * operator deletes them manually. After SCM restart, in-memory eviction state is lost, so + * completed archives persist until manual cleanup. + * + *

SCM restart while a job runs: the in-progress marker and {@code export_{jobId}/} + * remain on disk, but in-memory job status is lost. {@link #start()} treats the job as incomplete, + * removes the marker, workspace, and any partial {@code .tar.gz} for that job id, and the + * operator re-submits the export on the new leader. */ final class ExportFileManager { private static final Logger LOG = LoggerFactory.getLogger(ExportFileManager.class); + static final String IN_PROGRESS_MARKER_SUFFIX = ".in-progress"; - static final String EXPORT_JOB_DIR_PREFIX = "export-"; + static final String EXPORT_JOB_DIR_PREFIX = "export_"; + static final String EXPORT_ARCHIVE_SUFFIX = ".tar.gz"; + private final String exportDirectory; ExportFileManager(String exportDirectory) { @@ -66,8 +88,9 @@ void start() throws IOException { } String resolveTarPath(ExportScope scope, String fileTimestamp, String jobId) { - String tarFileName = String.format("container-ids-%s-%s-%s.tar", scope.getValue(), fileTimestamp, jobId); - return exportDirectory + File.separator + tarFileName; + String archiveFileName = String.format("container-ids-%s-%s-%s%s", + scope.getValue(), fileTimestamp, jobId, EXPORT_ARCHIVE_SUFFIX); + return exportDirectory + File.separator + archiveFileName; } void markExportInProgress(String jobId) throws IOException { @@ -84,7 +107,7 @@ void deleteExportTar(String tarPath) { } File tar = new File(tarPath); if (tar.isFile() && FileUtils.deleteQuietly(tar)) { - LOG.debug("Removed container export TAR: {}", tar.getName()); + LOG.debug("Removed container export archive: {}", tar.getName()); } } @@ -134,19 +157,19 @@ private void removeIncompleteExportArtifacts(String jobId) { File tar = findTarForJobId(jobId); if (tar != null) { FileUtils.deleteQuietly(tar); - LOG.info("Removed incomplete container export TAR for job {}: {}", jobId, tar.getName()); + LOG.info("Removed incomplete container export archive for job {}: {}", jobId, tar.getName()); } - File jobWorkDir = new File(exportDirectory, exportJobDirName(jobId)); - if (jobWorkDir.isDirectory()) { - FileUtils.deleteQuietly(jobWorkDir); - LOG.info("Removed orphaned container export work directory: {}", jobWorkDir.getAbsolutePath()); + File jobDir = new File(exportDirectory, exportJobDirName(jobId)); + if (jobDir.isDirectory()) { + FileUtils.deleteQuietly(jobDir); + LOG.info("Removed orphaned container export job directory: {}", jobDir.getAbsolutePath()); } } private File findTarForJobId(String jobId) { File exportDir = new File(exportDirectory); - File[] matches = exportDir.listFiles( - (dir, fileName) -> fileName.endsWith("-" + jobId + ".tar")); + String suffix = "-" + jobId + EXPORT_ARCHIVE_SUFFIX; + File[] matches = exportDir.listFiles((dir, fileName) -> fileName.endsWith(suffix)); if (matches == null || matches.length == 0) { return null; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java index c88cf886cc3f..806512f37a9b 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java @@ -24,7 +24,7 @@ * Container listing filters for an export job. * An export job filters containers by {@link ContainerHealthState}, {@link LifeCycleState} or both. * Example TAR name: - * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z-{jobId}.tar} + * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z-{jobId}.tar.gz} */ public final class ExportScope { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java index 248e6f0e36d8..922934f454c2 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -50,26 +50,28 @@ public void testResolveTarPath() { String jobId = UUID.randomUUID().toString(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); String tarPath = fileManager.resolveTarPath(scope, "20260101T120000Z", jobId); - assertTrue(tarPath.endsWith("container-ids-health-MISSING-20260101T120000Z-" + jobId + ".tar")); + assertTrue(tarPath.endsWith( + "container-ids-health-MISSING-20260101T120000Z-" + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); } @Test - public void testOrphanWorkDirRemovedOnStartup() throws Exception { + public void testOrphanJobDirRemovedOnStartup() throws Exception { String jobId = UUID.randomUUID().toString(); - Path orphan = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)).resolve("work"); - Files.createDirectories(orphan); + Path orphanJobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); + Files.createDirectories(orphanJobDir); fileManager.start(); - assertFalse(Files.exists(orphan)); + assertFalse(Files.exists(orphanJobDir)); } @Test public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { String jobId = UUID.randomUUID().toString(); - Path jobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)).resolve("work"); + Path jobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(jobDir); - File partialTar = new File(tempDir, "container-ids-health-MISSING-20260101T000000Z-" + jobId + ".tar"); + File partialTar = new File(tempDir, + "container-ids-health-MISSING-20260101T000000Z-" + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX); assertTrue(partialTar.createNewFile()); File inProgress = new File(tempDir, jobId + ExportFileManager.IN_PROGRESS_MARKER_SUFFIX); assertTrue(inProgress.createNewFile()); @@ -82,16 +84,17 @@ public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { } @Test - public void testOrphanWorkDirWithoutMarkerDoesNotDeleteCompletedTar() throws Exception { + public void testOrphanJobDirWithoutMarkerDoesNotDeleteCompletedTar() throws Exception { String jobId = UUID.randomUUID().toString(); - File completedTar = new File(tempDir, "container-ids-health-MISSING-20260101T000000Z-" + jobId + ".tar"); + File completedTar = new File(tempDir, + "container-ids-health-MISSING-20260101T000000Z-" + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX); assertTrue(completedTar.createNewFile()); - Path orphanWorkDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); - Files.createDirectories(orphanWorkDir.resolve("work")); + Path orphanJobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); + Files.createDirectories(orphanJobDir); fileManager.start(); assertTrue(completedTar.exists()); - assertFalse(Files.exists(orphanWorkDir)); + assertFalse(Files.exists(orphanJobDir)); } } From 3617ba63b981e0cf96538ccd512e7a7b11796747 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Tue, 28 Jul 2026 10:20:47 +0530 Subject: [PATCH 3/7] Use gz and file lock --- .../container/export/ExportFileManager.java | 151 +++++++++++++----- .../scm/container/export/ExportScope.java | 4 +- .../export/TestExportFileManager.java | 47 ++++-- 3 files changed, 148 insertions(+), 54 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java index bb30a0ef2a51..4ada8c1e1f35 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java @@ -19,12 +19,21 @@ import java.io.File; import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; import java.util.Objects; import java.util.UUID; import org.apache.commons.io.FileUtils; +import org.apache.ratis.util.AtomicFileOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,36 +42,45 @@ * *

The export directory ({@code exportDirectory}, typically {@code {scm.db.dirs}/exports}) * uses the layout below. The manager gzip-compresses the archive ({@code .tar.gz}) so operators - * can stream entries with {@code zcat} - * + * can stream entries with {@code zcat}. + * + *

While a job runs, shard text files are written under {@code export_{jobId}/}. The archive is + * created only after all shards are written. The export manager writes + * {@code container-ids-{scope}-{timestamp}.tar.gz.tmp} and atomically renames it to + * {@code .tar.gz} on close ({@link AtomicFileOutputStream}), so a partial {@code .tar.gz} is + * never visible. {@link #lock()} is used to exclude concurrent writers. + * * 

  * {exportDirectory}/
+ * ├── in_use.lock
  * ├── {jobId}.in-progress
- * ├── container-ids-{scope}-{timestamp}-{jobId}.tar.gz
+ * ├── container-ids-{scope}-{timestamp}.tar.gz
+ * ├── container-ids-{scope}-{timestamp}.tar.gz.tmp
  * └── export_{jobId}/
  *     ├── container-ids-{scope}-{timestamp}-part001.txt
  *     └── ...
  *
* - *

{@code export_{jobId}/} holds shard text files while the job appends them into the archive. - * *

When {@code export_{jobId}/} is deleted: the export manager deletes it after the - * archive closes successfully, or during {@link #cleanupFailedArtifacts} on failure or cancel. + * archive is committed, or during {@link #cleanupFailedArtifacts} on failure or cancel. * On startup, {@link #start()} deletes a leftover {@code export_{jobId}/} when no in-progress * marker remains. If the marker still exists, {@link #start()} deletes {@code export_{jobId}/} - * together with the marker and any partial archive for that job id. + * together with the marker and any {@code .tar.gz.tmp} for that incomplete job. + * + *

When {@code .tar.gz.tmp} is deleted: only the temporary file is removed for + * incomplete work; a partial {@code .tar.gz} is never written. {@link #cleanupFailedArtifacts} + * deletes {@code .tar.gz.tmp} for failed or cancelled jobs. {@link #start()} deletes + * {@code .tar.gz.tmp} for jobs that still have an in-progress marker. * - *

When {@code .tar.gz} is deleted: {@link #cleanupFailedArtifacts} deletes partial - * archives for failed or cancelled jobs. {@link #start()} deletes partial archives for jobs that - * still have an in-progress marker. Completed archives remain on disk until the export manager - * evicts the job from memory ({@code maxTerminalJobs} in {@code ContainerExportManager}) or an - * operator deletes them manually. After SCM restart, in-memory eviction state is lost, so - * completed archives persist until manual cleanup. + *

When completed {@code .tar.gz} is deleted: completed archives remain on disk until + * the export manager evicts them ({@code maxTerminalJobs} in {@code ContainerExportManager}). * - *

SCM restart while a job runs: the in-progress marker and {@code export_{jobId}/} - * remain on disk, but in-memory job status is lost. {@link #start()} treats the job as incomplete, - * removes the marker, workspace, and any partial {@code .tar.gz} for that job id, and the - * operator re-submits the export on the new leader. + *

SCM restart: in-memory job status is lost and {@code jobId} cannot be recovered from + * the archive file name. {@link #listCompletedArchivePaths()} returns existing {@code tarPath} + * values (oldest first) so {@code ContainerExportManager} can rebuild terminal-job eviction state. + * Jobs with an in-progress marker are treated as incomplete: {@link #start()} removes the marker, + * {@code export_{jobId}/}, and any {@code .tar.gz.tmp}, and the operator re-submits the export + * on the new leader. */ final class ExportFileManager { @@ -71,8 +89,11 @@ final class ExportFileManager { static final String IN_PROGRESS_MARKER_SUFFIX = ".in-progress"; static final String EXPORT_JOB_DIR_PREFIX = "export_"; static final String EXPORT_ARCHIVE_SUFFIX = ".tar.gz"; + static final String EXPORT_ARCHIVE_TMP_SUFFIX = EXPORT_ARCHIVE_SUFFIX + AtomicFileOutputStream.TMP_EXTENSION; + static final String EXPORT_LOCK_NAME = "in_use.lock"; private final String exportDirectory; + private FileLock exportDirectoryLock; ExportFileManager(String exportDirectory) { this.exportDirectory = Objects.requireNonNull(exportDirectory, "exportDirectory == null"); @@ -87,10 +108,60 @@ void start() throws IOException { cleanupOrphanedExportArtifacts(); } - String resolveTarPath(ExportScope scope, String fileTimestamp, String jobId) { - String archiveFileName = String.format("container-ids-%s-%s-%s%s", - scope.getValue(), fileTimestamp, jobId, EXPORT_ARCHIVE_SUFFIX); - return exportDirectory + File.separator + archiveFileName; + void lock() throws IOException { + if (exportDirectoryLock != null) { + return; + } + File lockFile = new File(exportDirectory, EXPORT_LOCK_NAME); + RandomAccessFile lockAccessFile = new RandomAccessFile(lockFile, "rws"); + try { + FileLock lock = lockAccessFile.getChannel().tryLock(); + if (lock == null) { + lockAccessFile.close(); + throw new OverlappingFileLockException(); + } + exportDirectoryLock = lock; + LOG.debug("Acquired container export directory lock {}", lockFile.getAbsolutePath()); + } catch (OverlappingFileLockException | IOException e) { + lockAccessFile.close(); + throw new IOException("Failed to lock container export directory " + exportDirectory, e); + } + } + + void unlock() throws IOException { + if (exportDirectoryLock == null) { + return; + } + exportDirectoryLock.release(); + exportDirectoryLock.channel().close(); + exportDirectoryLock = null; + } + + File resolveArchiveFile(ExportScope scope, String fileTimestamp) { + return new File(exportDirectory, + String.format("container-ids-%s-%s%s", scope.getValue(), fileTimestamp, EXPORT_ARCHIVE_SUFFIX)); + } + + File resolveArchiveTempFile(ExportScope scope, String fileTimestamp) { + return AtomicFileOutputStream.getTemporaryFile(resolveArchiveFile(scope, fileTimestamp)); + } + + /** + * Returns completed archive paths ({@code tarPath} in {@code ExportJob.Status}), oldest first. + */ + List listCompletedArchivePaths() { + File exportDir = new File(exportDirectory); + File[] matches = exportDir.listFiles((dir, fileName) -> fileName.endsWith(EXPORT_ARCHIVE_SUFFIX) + && !fileName.endsWith(EXPORT_ARCHIVE_TMP_SUFFIX)); + if (matches == null || matches.length == 0) { + return Collections.emptyList(); + } + Arrays.sort(matches, Comparator.comparingLong(File::lastModified)); + List archivePaths = new ArrayList<>(matches.length); + for (File archive : matches) { + archivePaths.add(archive.getAbsolutePath()); + } + return archivePaths; } void markExportInProgress(String jobId) throws IOException { @@ -105,18 +176,20 @@ void deleteExportTar(String tarPath) { if (tarPath == null) { return; } - File tar = new File(tarPath); - if (tar.isFile() && FileUtils.deleteQuietly(tar)) { - LOG.debug("Removed container export archive: {}", tar.getName()); + File archive = new File(tarPath); + if (archive.isFile() && FileUtils.deleteQuietly(archive)) { + LOG.debug("Removed container export archive: {}", archive.getName()); } + FileUtils.deleteQuietly(AtomicFileOutputStream.getTemporaryFile(archive)); } - void cleanupFailedArtifacts(Path jobDir, File tarFile, String jobId) { + void cleanupFailedArtifacts(Path jobDir, File archiveFile, String jobId) { if (jobDir != null) { FileUtils.deleteQuietly(jobDir.toFile()); } - if (tarFile != null) { - FileUtils.deleteQuietly(tarFile); + if (archiveFile != null) { + FileUtils.deleteQuietly(AtomicFileOutputStream.getTemporaryFile(archiveFile)); + FileUtils.deleteQuietly(archiveFile); } clearExportInProgress(jobId); } @@ -149,31 +222,31 @@ private void cleanupOrphanedExportArtifacts() { } } } + deleteOrphanArchiveTempFiles(); } private void removeIncompleteExportArtifacts(String jobId) { LOG.info("Removing incomplete container export artifacts for job {}", jobId); FileUtils.deleteQuietly(inProgressMarkerFile(jobId)); - File tar = findTarForJobId(jobId); - if (tar != null) { - FileUtils.deleteQuietly(tar); - LOG.info("Removed incomplete container export archive for job {}: {}", jobId, tar.getName()); - } + deleteOrphanArchiveTempFiles(); File jobDir = new File(exportDirectory, exportJobDirName(jobId)); if (jobDir.isDirectory()) { FileUtils.deleteQuietly(jobDir); - LOG.info("Removed orphaned container export job directory: {}", jobDir.getAbsolutePath()); + LOG.debug("Removed orphaned container export job directory: {}", jobDir.getAbsolutePath()); } } - private File findTarForJobId(String jobId) { + private void deleteOrphanArchiveTempFiles() { File exportDir = new File(exportDirectory); - String suffix = "-" + jobId + EXPORT_ARCHIVE_SUFFIX; - File[] matches = exportDir.listFiles((dir, fileName) -> fileName.endsWith(suffix)); - if (matches == null || matches.length == 0) { - return null; + File[] tempFiles = exportDir.listFiles((dir, fileName) -> fileName.endsWith(EXPORT_ARCHIVE_TMP_SUFFIX)); + if (tempFiles == null) { + return; + } + for (File tempFile : tempFiles) { + if (FileUtils.deleteQuietly(tempFile)) { + LOG.debug("Removed incomplete container export archive temp file: {}", tempFile.getName()); + } } - return matches[0]; } private File inProgressMarkerFile(String jobId) { diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java index 806512f37a9b..6cdee3992416 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java @@ -23,8 +23,8 @@ /** * Container listing filters for an export job. * An export job filters containers by {@link ContainerHealthState}, {@link LifeCycleState} or both. - * Example TAR name: - * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z-{jobId}.tar.gz} + * Example archive name: + * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z.tar.gz} */ public final class ExportScope { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java index 922934f454c2..dffbdc695e19 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -17,6 +17,7 @@ package org.apache.hadoop.hdds.scm.container.export; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -46,12 +47,32 @@ public void setup() throws Exception { } @Test - public void testResolveTarPath() { - String jobId = UUID.randomUUID().toString(); + public void testResolveArchiveFile() { + ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); + File archive = fileManager.resolveArchiveFile(scope, "20260101T120000Z"); + assertTrue(archive.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); + } + + @Test + public void testResolveArchiveTempFile() { ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - String tarPath = fileManager.resolveTarPath(scope, "20260101T120000Z", jobId); - assertTrue(tarPath.endsWith( - "container-ids-health-MISSING-20260101T120000Z-" + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); + File tempFile = fileManager.resolveArchiveTempFile(scope, "20260101T120000Z"); + assertTrue(tempFile.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_TMP_SUFFIX)); + } + + @Test + public void testListCompletedArchivePaths() throws Exception { + ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); + File olderArchive = fileManager.resolveArchiveFile(scope, "20260101T120000Z"); + assertTrue(olderArchive.createNewFile()); + File newerArchive = fileManager.resolveArchiveFile(scope, "20260101T120001Z"); + assertTrue(newerArchive.createNewFile()); + File tempArchive = fileManager.resolveArchiveTempFile(scope, "20260101T120002Z"); + assertTrue(tempArchive.createNewFile()); + + assertEquals(2, fileManager.listCompletedArchivePaths().size()); + assertEquals(olderArchive.getAbsolutePath(), fileManager.listCompletedArchivePaths().get(0)); + assertEquals(newerArchive.getAbsolutePath(), fileManager.listCompletedArchivePaths().get(1)); } @Test @@ -70,31 +91,31 @@ public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { String jobId = UUID.randomUUID().toString(); Path jobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(jobDir); - File partialTar = new File(tempDir, - "container-ids-health-MISSING-20260101T000000Z-" + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX); - assertTrue(partialTar.createNewFile()); + ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); + File partialArchiveTemp = fileManager.resolveArchiveTempFile(scope, "20260101T000000Z"); + assertTrue(partialArchiveTemp.createNewFile()); File inProgress = new File(tempDir, jobId + ExportFileManager.IN_PROGRESS_MARKER_SUFFIX); assertTrue(inProgress.createNewFile()); fileManager.start(); assertFalse(Files.exists(jobDir)); - assertFalse(partialTar.exists()); + assertFalse(partialArchiveTemp.exists()); assertFalse(inProgress.exists()); } @Test public void testOrphanJobDirWithoutMarkerDoesNotDeleteCompletedTar() throws Exception { String jobId = UUID.randomUUID().toString(); - File completedTar = new File(tempDir, - "container-ids-health-MISSING-20260101T000000Z-" + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX); - assertTrue(completedTar.createNewFile()); + ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); + File completedArchive = fileManager.resolveArchiveFile(scope, "20260101T000000Z"); + assertTrue(completedArchive.createNewFile()); Path orphanJobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(orphanJobDir); fileManager.start(); - assertTrue(completedTar.exists()); + assertTrue(completedArchive.exists()); assertFalse(Files.exists(orphanJobDir)); } } From 915262fd8d8e604524ceb1e4f5b5b57b0c5e9e66 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Wed, 29 Jul 2026 10:25:47 +0530 Subject: [PATCH 4/7] Fix test failure in TestExportFileManager.testListCompletedArchivePaths --- .../scm/container/export/TestExportFileManager.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java index dffbdc695e19..3dedd6342c58 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -24,6 +24,7 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.UUID; import org.apache.hadoop.hdds.scm.container.ContainerHealthState; import org.junit.jupiter.api.BeforeEach; @@ -65,14 +66,17 @@ public void testListCompletedArchivePaths() throws Exception { ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); File olderArchive = fileManager.resolveArchiveFile(scope, "20260101T120000Z"); assertTrue(olderArchive.createNewFile()); + assertTrue(olderArchive.setLastModified(1_000L)); File newerArchive = fileManager.resolveArchiveFile(scope, "20260101T120001Z"); assertTrue(newerArchive.createNewFile()); + assertTrue(newerArchive.setLastModified(2_000L)); File tempArchive = fileManager.resolveArchiveTempFile(scope, "20260101T120002Z"); assertTrue(tempArchive.createNewFile()); - assertEquals(2, fileManager.listCompletedArchivePaths().size()); - assertEquals(olderArchive.getAbsolutePath(), fileManager.listCompletedArchivePaths().get(0)); - assertEquals(newerArchive.getAbsolutePath(), fileManager.listCompletedArchivePaths().get(1)); + List completedPaths = fileManager.listCompletedArchivePaths(); + assertEquals(2, completedPaths.size()); + assertEquals(olderArchive.getAbsolutePath(), completedPaths.get(0)); + assertEquals(newerArchive.getAbsolutePath(), completedPaths.get(1)); } @Test From e49bb6006edf19d11291ab4c2d738b0e460e51c2 Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Thu, 30 Jul 2026 11:20:36 +0530 Subject: [PATCH 5/7] Include jobId and remove in-progress marker --- .../container/export/ExportFileManager.java | 129 ++++++------------ .../scm/container/export/ExportScope.java | 2 +- .../export/TestExportFileManager.java | 38 ++++-- 3 files changed, 69 insertions(+), 100 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java index 4ada8c1e1f35..c7c2cb7722f2 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java @@ -46,48 +46,38 @@ * *

While a job runs, shard text files are written under {@code export_{jobId}/}. The archive is * created only after all shards are written. The export manager writes - * {@code container-ids-{scope}-{timestamp}.tar.gz.tmp} and atomically renames it to + * {@code container-ids-{scope}-{timestamp}_job{jobId}.tar.gz.tmp} and atomically renames it to * {@code .tar.gz} on close ({@link AtomicFileOutputStream}), so a partial {@code .tar.gz} is - * never visible. {@link #lock()} is used to exclude concurrent writers. + * never visible. {@link #lock()} uses {@code in_use.lock} to exclude concurrent writers. * * 

  * {exportDirectory}/
  * ├── in_use.lock
- * ├── {jobId}.in-progress
- * ├── container-ids-{scope}-{timestamp}.tar.gz
- * ├── container-ids-{scope}-{timestamp}.tar.gz.tmp
+ * ├── container-ids-{scope}-{timestamp}_job{jobId}.tar.gz
+ * ├── container-ids-{scope}-{timestamp}_job{jobId}.tar.gz.tmp
  * └── export_{jobId}/
  *     ├── container-ids-{scope}-{timestamp}-part001.txt
  *     └── ...
  *
* - *

When {@code export_{jobId}/} is deleted: the export manager deletes it after the - * archive is committed, or during {@link #cleanupFailedArtifacts} on failure or cancel. - * On startup, {@link #start()} deletes a leftover {@code export_{jobId}/} when no in-progress - * marker remains. If the marker still exists, {@link #start()} deletes {@code export_{jobId}/} - * together with the marker and any {@code .tar.gz.tmp} for that incomplete job. + *

Incomplete work ({@code export_{jobId}/} and {@code .tar.gz.tmp}) is removed by + * {@link #cleanupFailedJob(Path, File)} on failure or cancel, and by {@link #start()} for every + * leftover directory and temp file after SCM restart. Completed {@code .tar.gz} files are kept. * - *

When {@code .tar.gz.tmp} is deleted: only the temporary file is removed for - * incomplete work; a partial {@code .tar.gz} is never written. {@link #cleanupFailedArtifacts} - * deletes {@code .tar.gz.tmp} for failed or cancelled jobs. {@link #start()} deletes - * {@code .tar.gz.tmp} for jobs that still have an in-progress marker. + *

Completed {@code .tar.gz} remains on disk until the export manager evicts it + * ({@code maxTerminalJobs} in {@code ContainerExportManager}) via {@link #deleteExportTar(String)}. * - *

When completed {@code .tar.gz} is deleted: completed archives remain on disk until - * the export manager evicts them ({@code maxTerminalJobs} in {@code ContainerExportManager}). - * - *

SCM restart: in-memory job status is lost and {@code jobId} cannot be recovered from - * the archive file name. {@link #listCompletedArchivePaths()} returns existing {@code tarPath} - * values (oldest first) so {@code ContainerExportManager} can rebuild terminal-job eviction state. - * Jobs with an in-progress marker are treated as incomplete: {@link #start()} removes the marker, - * {@code export_{jobId}/}, and any {@code .tar.gz.tmp}, and the operator re-submits the export - * on the new leader. + *

SCM restart: in-memory job status is lost. {@link #start()} clears incomplete work; + * {@link #listCompletedArchivePaths()} returns existing {@code tarPath} values (oldest first); + * {@link #jobIdFromArchiveFileName(String)} parses {@code jobId} for terminal-job rebuild in + * {@code ContainerExportManager}. */ final class ExportFileManager { private static final Logger LOG = LoggerFactory.getLogger(ExportFileManager.class); - static final String IN_PROGRESS_MARKER_SUFFIX = ".in-progress"; static final String EXPORT_JOB_DIR_PREFIX = "export_"; + static final String EXPORT_ARCHIVE_JOB_INFIX = "_job"; static final String EXPORT_ARCHIVE_SUFFIX = ".tar.gz"; static final String EXPORT_ARCHIVE_TMP_SUFFIX = EXPORT_ARCHIVE_SUFFIX + AtomicFileOutputStream.TMP_EXTENSION; static final String EXPORT_LOCK_NAME = "in_use.lock"; @@ -105,7 +95,7 @@ String getExportDirectory() { void start() throws IOException { Files.createDirectories(Paths.get(exportDirectory)); - cleanupOrphanedExportArtifacts(); + removeIncompleteWorkOnStartup(); } void lock() throws IOException { @@ -137,13 +127,13 @@ void unlock() throws IOException { exportDirectoryLock = null; } - File resolveArchiveFile(ExportScope scope, String fileTimestamp) { - return new File(exportDirectory, - String.format("container-ids-%s-%s%s", scope.getValue(), fileTimestamp, EXPORT_ARCHIVE_SUFFIX)); + File resolveArchiveFile(ExportScope scope, String fileTimestamp, String jobId) { + return new File(exportDirectory, String.format("container-ids-%s-%s%s%s%s", + scope.getValue(), fileTimestamp, EXPORT_ARCHIVE_JOB_INFIX, jobId, EXPORT_ARCHIVE_SUFFIX)); } - File resolveArchiveTempFile(ExportScope scope, String fileTimestamp) { - return AtomicFileOutputStream.getTemporaryFile(resolveArchiveFile(scope, fileTimestamp)); + File resolveArchiveTempFile(ExportScope scope, String fileTimestamp, String jobId) { + return AtomicFileOutputStream.getTemporaryFile(resolveArchiveFile(scope, fileTimestamp, jobId)); } /** @@ -164,12 +154,17 @@ List listCompletedArchivePaths() { return archivePaths; } - void markExportInProgress(String jobId) throws IOException { - Files.createFile(inProgressMarkerFile(jobId).toPath()); - } - - void clearExportInProgress(String jobId) { - FileUtils.deleteQuietly(inProgressMarkerFile(jobId)); + static String jobIdFromArchiveFileName(String fileName) { + if (!fileName.endsWith(EXPORT_ARCHIVE_SUFFIX)) { + return null; + } + String nameWithoutSuffix = fileName.substring(0, fileName.length() - EXPORT_ARCHIVE_SUFFIX.length()); + int jobIndex = nameWithoutSuffix.lastIndexOf(EXPORT_ARCHIVE_JOB_INFIX); + if (jobIndex < 0) { + return null; + } + String jobId = nameWithoutSuffix.substring(jobIndex + EXPORT_ARCHIVE_JOB_INFIX.length()); + return isUuidDirectoryName(jobId) ? jobId : null; } void deleteExportTar(String tarPath) { @@ -183,76 +178,36 @@ void deleteExportTar(String tarPath) { FileUtils.deleteQuietly(AtomicFileOutputStream.getTemporaryFile(archive)); } - void cleanupFailedArtifacts(Path jobDir, File archiveFile, String jobId) { + void cleanupFailedJob(Path jobDir, File archiveFile) { if (jobDir != null) { FileUtils.deleteQuietly(jobDir.toFile()); } if (archiveFile != null) { FileUtils.deleteQuietly(AtomicFileOutputStream.getTemporaryFile(archiveFile)); - FileUtils.deleteQuietly(archiveFile); } - clearExportInProgress(jobId); } - private void cleanupOrphanedExportArtifacts() { + private void removeIncompleteWorkOnStartup() { File exportDir = new File(exportDirectory); File[] children = exportDir.listFiles(); - if (children == null) { - return; - } - for (File child : children) { - if (child.isFile() && child.getName().endsWith(IN_PROGRESS_MARKER_SUFFIX)) { - String jobId = child.getName().substring( - 0, child.getName().length() - IN_PROGRESS_MARKER_SUFFIX.length()); - if (isUuidDirectoryName(jobId)) { - removeIncompleteExportArtifacts(jobId); - } - } - } - for (File child : children) { - if (child.isDirectory()) { - String jobId = jobIdFromExportDirName(child.getName()); - if (jobId == null) { - continue; - } - if (inProgressMarkerFile(jobId).exists()) { - removeIncompleteExportArtifacts(jobId); - } else { + if (children != null) { + for (File child : children) { + if (child.isDirectory() && jobIdFromExportDirName(child.getName()) != null) { FileUtils.deleteQuietly(child); + LOG.debug("Removed incomplete container export job directory: {}", child.getAbsolutePath()); } } } - deleteOrphanArchiveTempFiles(); - } - - private void removeIncompleteExportArtifacts(String jobId) { - LOG.info("Removing incomplete container export artifacts for job {}", jobId); - FileUtils.deleteQuietly(inProgressMarkerFile(jobId)); - deleteOrphanArchiveTempFiles(); - File jobDir = new File(exportDirectory, exportJobDirName(jobId)); - if (jobDir.isDirectory()) { - FileUtils.deleteQuietly(jobDir); - LOG.debug("Removed orphaned container export job directory: {}", jobDir.getAbsolutePath()); - } - } - - private void deleteOrphanArchiveTempFiles() { - File exportDir = new File(exportDirectory); File[] tempFiles = exportDir.listFiles((dir, fileName) -> fileName.endsWith(EXPORT_ARCHIVE_TMP_SUFFIX)); - if (tempFiles == null) { - return; - } - for (File tempFile : tempFiles) { - if (FileUtils.deleteQuietly(tempFile)) { - LOG.debug("Removed incomplete container export archive temp file: {}", tempFile.getName()); + if (tempFiles != null) { + for (File tempFile : tempFiles) { + if (FileUtils.deleteQuietly(tempFile)) { + LOG.debug("Removed incomplete container export archive temp file: {}", tempFile.getName()); + } } } } - private File inProgressMarkerFile(String jobId) { - return new File(exportDirectory, jobId + IN_PROGRESS_MARKER_SUFFIX); - } - static String exportJobDirName(String jobId) { return EXPORT_JOB_DIR_PREFIX + jobId; } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java index 6cdee3992416..975f93c08e74 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java @@ -24,7 +24,7 @@ * Container listing filters for an export job. * An export job filters containers by {@link ContainerHealthState}, {@link LifeCycleState} or both. * Example archive name: - * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z.tar.gz} + * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z_job{jobId}.tar.gz} */ public final class ExportScope { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java index 3dedd6342c58..247165ef934f 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; @@ -49,28 +50,44 @@ public void setup() throws Exception { @Test public void testResolveArchiveFile() { + String jobId = UUID.randomUUID().toString(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - File archive = fileManager.resolveArchiveFile(scope, "20260101T120000Z"); - assertTrue(archive.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); + File archive = fileManager.resolveArchiveFile(scope, "20260101T120000Z", jobId); + assertTrue(archive.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId + + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); } @Test public void testResolveArchiveTempFile() { + String jobId = UUID.randomUUID().toString(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - File tempFile = fileManager.resolveArchiveTempFile(scope, "20260101T120000Z"); + File tempFile = fileManager.resolveArchiveTempFile(scope, "20260101T120000Z", jobId); assertTrue(tempFile.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_TMP_SUFFIX)); } + @Test + public void testJobIdFromArchiveFileName() { + String jobId = UUID.randomUUID().toString(); + String fileName = "container-ids-health-MISSING-20260101T120000Z" + + ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX; + assertEquals(jobId, ExportFileManager.jobIdFromArchiveFileName(fileName)); + assertNull(ExportFileManager.jobIdFromArchiveFileName("container-ids-health-MISSING-20260101T120000Z" + + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); + } + @Test public void testListCompletedArchivePaths() throws Exception { ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - File olderArchive = fileManager.resolveArchiveFile(scope, "20260101T120000Z"); + String olderJobId = UUID.randomUUID().toString(); + File olderArchive = fileManager.resolveArchiveFile(scope, "20260101T120000Z", olderJobId); assertTrue(olderArchive.createNewFile()); assertTrue(olderArchive.setLastModified(1_000L)); - File newerArchive = fileManager.resolveArchiveFile(scope, "20260101T120001Z"); + String newerJobId = UUID.randomUUID().toString(); + File newerArchive = fileManager.resolveArchiveFile(scope, "20260101T120001Z", newerJobId); assertTrue(newerArchive.createNewFile()); assertTrue(newerArchive.setLastModified(2_000L)); - File tempArchive = fileManager.resolveArchiveTempFile(scope, "20260101T120002Z"); + String tempJobId = UUID.randomUUID().toString(); + File tempArchive = fileManager.resolveArchiveTempFile(scope, "20260101T120002Z", tempJobId); assertTrue(tempArchive.createNewFile()); List completedPaths = fileManager.listCompletedArchivePaths(); @@ -96,23 +113,20 @@ public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { Path jobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(jobDir); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - File partialArchiveTemp = fileManager.resolveArchiveTempFile(scope, "20260101T000000Z"); + File partialArchiveTemp = fileManager.resolveArchiveTempFile(scope, "20260101T000000Z", jobId); assertTrue(partialArchiveTemp.createNewFile()); - File inProgress = new File(tempDir, jobId + ExportFileManager.IN_PROGRESS_MARKER_SUFFIX); - assertTrue(inProgress.createNewFile()); fileManager.start(); assertFalse(Files.exists(jobDir)); assertFalse(partialArchiveTemp.exists()); - assertFalse(inProgress.exists()); } @Test - public void testOrphanJobDirWithoutMarkerDoesNotDeleteCompletedTar() throws Exception { + public void testOrphanJobDirDoesNotDeleteCompletedTar() throws Exception { String jobId = UUID.randomUUID().toString(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - File completedArchive = fileManager.resolveArchiveFile(scope, "20260101T000000Z"); + File completedArchive = fileManager.resolveArchiveFile(scope, "20260101T000000Z", jobId); assertTrue(completedArchive.createNewFile()); Path orphanJobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(orphanJobDir); From 84a4c031fbd0b13dd398594d5c73d182f750b4fb Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Sat, 1 Aug 2026 08:58:16 +0530 Subject: [PATCH 6/7] Create ExportJob.Id and use timestamp in filename --- .../apache/hadoop/ozone/util/UUIDUtil.java | 9 +++ .../container/export/ExportFileManager.java | 44 ++++++----- .../hdds/scm/container/export/ExportJob.java | 74 +++++++++++++++++++ .../scm/container/export/ExportScope.java | 16 ++-- .../export/TestExportFileManager.java | 45 +++++++---- 5 files changed, 143 insertions(+), 45 deletions(-) create mode 100644 hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportJob.java diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDUtil.java index 8f4da0cfc46a..ef1de0d6aa9d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/util/UUIDUtil.java @@ -19,6 +19,7 @@ import java.security.SecureRandom; import java.util.Random; +import java.util.UUID; import java.util.function.Consumer; /** @@ -47,6 +48,14 @@ private static byte[] getUUIDBytes(Consumer generator) { return bytes; } + public static boolean isValidUuidString(String value) { + try { + return value.equals(UUID.fromString(value).toString()); + } catch (IllegalArgumentException e) { + return false; + } + } + private UUIDUtil() { } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java index c7c2cb7722f2..f75a6e79ddaf 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java @@ -31,8 +31,8 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; -import java.util.UUID; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.ozone.util.UUIDUtil; import org.apache.ratis.util.AtomicFileOutputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,6 +81,7 @@ final class ExportFileManager { static final String EXPORT_ARCHIVE_SUFFIX = ".tar.gz"; static final String EXPORT_ARCHIVE_TMP_SUFFIX = EXPORT_ARCHIVE_SUFFIX + AtomicFileOutputStream.TMP_EXTENSION; static final String EXPORT_LOCK_NAME = "in_use.lock"; + private static final int ARCHIVE_TIMESTAMP_LENGTH = 16; private final String exportDirectory; private FileLock exportDirectoryLock; @@ -127,13 +128,13 @@ void unlock() throws IOException { exportDirectoryLock = null; } - File resolveArchiveFile(ExportScope scope, String fileTimestamp, String jobId) { + File resolveArchiveFile(ExportScope scope, String archiveTimestamp, ExportJob.Id jobId) { return new File(exportDirectory, String.format("container-ids-%s-%s%s%s%s", - scope.getValue(), fileTimestamp, EXPORT_ARCHIVE_JOB_INFIX, jobId, EXPORT_ARCHIVE_SUFFIX)); + scope.getValue(), archiveTimestamp, EXPORT_ARCHIVE_JOB_INFIX, jobId.getValue(), EXPORT_ARCHIVE_SUFFIX)); } - File resolveArchiveTempFile(ExportScope scope, String fileTimestamp, String jobId) { - return AtomicFileOutputStream.getTemporaryFile(resolveArchiveFile(scope, fileTimestamp, jobId)); + File resolveArchiveTempFile(ExportScope scope, String archiveTimestamp, ExportJob.Id jobId) { + return AtomicFileOutputStream.getTemporaryFile(resolveArchiveFile(scope, archiveTimestamp, jobId)); } /** @@ -146,7 +147,8 @@ List listCompletedArchivePaths() { if (matches == null || matches.length == 0) { return Collections.emptyList(); } - Arrays.sort(matches, Comparator.comparingLong(File::lastModified)); + Arrays.sort(matches, Comparator.comparing( + file -> archiveTimestampFromArchiveFileName(file.getName()))); List archivePaths = new ArrayList<>(matches.length); for (File archive : matches) { archivePaths.add(archive.getAbsolutePath()); @@ -154,7 +156,17 @@ List listCompletedArchivePaths() { return archivePaths; } - static String jobIdFromArchiveFileName(String fileName) { + static String archiveTimestampFromArchiveFileName(String fileName) { + int jobIndex = fileName.lastIndexOf(EXPORT_ARCHIVE_JOB_INFIX); + if (jobIndex < ARCHIVE_TIMESTAMP_LENGTH + 1 + || !fileName.endsWith(EXPORT_ARCHIVE_SUFFIX) + || fileName.endsWith(EXPORT_ARCHIVE_TMP_SUFFIX)) { + return null; + } + return fileName.substring(jobIndex - ARCHIVE_TIMESTAMP_LENGTH, jobIndex); + } + + static ExportJob.Id jobIdFromArchiveFileName(String fileName) { if (!fileName.endsWith(EXPORT_ARCHIVE_SUFFIX)) { return null; } @@ -164,7 +176,7 @@ static String jobIdFromArchiveFileName(String fileName) { return null; } String jobId = nameWithoutSuffix.substring(jobIndex + EXPORT_ARCHIVE_JOB_INFIX.length()); - return isUuidDirectoryName(jobId) ? jobId : null; + return UUIDUtil.isValidUuidString(jobId) ? ExportJob.Id.of(jobId) : null; } void deleteExportTar(String tarPath) { @@ -208,23 +220,15 @@ private void removeIncompleteWorkOnStartup() { } } - static String exportJobDirName(String jobId) { - return EXPORT_JOB_DIR_PREFIX + jobId; + static String exportJobDirName(ExportJob.Id jobId) { + return EXPORT_JOB_DIR_PREFIX + jobId.getValue(); } - private static String jobIdFromExportDirName(String dirName) { + private static ExportJob.Id jobIdFromExportDirName(String dirName) { if (!dirName.startsWith(EXPORT_JOB_DIR_PREFIX)) { return null; } String jobId = dirName.substring(EXPORT_JOB_DIR_PREFIX.length()); - return isUuidDirectoryName(jobId) ? jobId : null; - } - - private static boolean isUuidDirectoryName(String directoryName) { - try { - return directoryName.equals(UUID.fromString(directoryName).toString()); - } catch (IllegalArgumentException e) { - return false; - } + return UUIDUtil.isValidUuidString(jobId) ? ExportJob.Id.of(jobId) : null; } } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportJob.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportJob.java new file mode 100644 index 000000000000..f9dee07eaea1 --- /dev/null +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportJob.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.container.export; + +import java.util.Objects; +import java.util.UUID; + +/** + * Container ID export job identifier. + */ +public final class ExportJob { + + /** + * Unique job identifier. + */ + public static final class Id { + private final String value; + + private Id(String value) { + this.value = Objects.requireNonNull(value, "value == null"); + } + + public static Id newId() { + return new Id(UUID.randomUUID().toString()); + } + + public static Id of(String value) { + return new Id(value); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return value; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof Id)) { + return false; + } + return value.equals(((Id) obj).value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + } + + private ExportJob() { + } +} diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java index 975f93c08e74..7c488ab799eb 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java @@ -28,6 +28,7 @@ */ public final class ExportScope { + private static final String ANY = "ANY"; private final LifeCycleState lifeCycleState; private final ContainerHealthState healthState; private final String value; @@ -39,17 +40,10 @@ private ExportScope(LifeCycleState lifeCycleState, ContainerHealthState healthSt } public static ExportScope of(LifeCycleState lifeCycleState, ContainerHealthState healthState) { - StringBuilder sb = new StringBuilder(); - if (healthState != null) { - sb.append("health-").append(healthState.name()); - } - if (lifeCycleState != null) { - if (sb.length() > 0) { - sb.append('_'); - } - sb.append("lifecycle-").append(lifeCycleState.name()); - } - return new ExportScope(lifeCycleState, healthState, sb.toString()); + String health = healthState != null ? healthState.name() : ANY; + String lifecycle = lifeCycleState != null ? lifeCycleState.name() : ANY; + String value = "health-" + health + "_lifecycle-" + lifecycle; + return new ExportScope(lifeCycleState, healthState, value); } public LifeCycleState getLifeCycleState() { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java index 247165ef934f..67e3d5f17818 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -27,6 +27,7 @@ import java.nio.file.Path; import java.util.List; import java.util.UUID; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.scm.container.ContainerHealthState; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,18 +49,27 @@ public void setup() throws Exception { fileManager.start(); } + @Test + public void testExportScopeUsesAnyForNullFilters() { + assertEquals("health-MISSING_lifecycle-ANY", + ExportScope.of(null, ContainerHealthState.MISSING).getValue()); + assertEquals("health-ANY_lifecycle-OPEN", + ExportScope.of(LifeCycleState.OPEN, null).getValue()); + } + @Test public void testResolveArchiveFile() { - String jobId = UUID.randomUUID().toString(); + ExportJob.Id jobId = ExportJob.Id.newId(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); File archive = fileManager.resolveArchiveFile(scope, "20260101T120000Z", jobId); - assertTrue(archive.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId + assertTrue(archive.getName().contains("health-MISSING_lifecycle-ANY-20260101T120000Z")); + assertTrue(archive.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId.getValue() + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); } @Test public void testResolveArchiveTempFile() { - String jobId = UUID.randomUUID().toString(); + ExportJob.Id jobId = ExportJob.Id.newId(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); File tempFile = fileManager.resolveArchiveTempFile(scope, "20260101T120000Z", jobId); assertTrue(tempFile.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_TMP_SUFFIX)); @@ -68,25 +78,32 @@ public void testResolveArchiveTempFile() { @Test public void testJobIdFromArchiveFileName() { String jobId = UUID.randomUUID().toString(); - String fileName = "container-ids-health-MISSING-20260101T120000Z" + String fileName = "container-ids-health-MISSING_lifecycle-ANY-20260101T120000Z" + ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX; - assertEquals(jobId, ExportFileManager.jobIdFromArchiveFileName(fileName)); - assertNull(ExportFileManager.jobIdFromArchiveFileName("container-ids-health-MISSING-20260101T120000Z" + assertEquals(ExportJob.Id.of(jobId), ExportFileManager.jobIdFromArchiveFileName(fileName)); + assertNull(ExportFileManager.jobIdFromArchiveFileName("container-ids-health-MISSING_lifecycle-ANY-20260101T120000Z" + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); } + @Test + public void testArchiveTimestampFromArchiveFileName() { + String fileName = "container-ids-health-MISSING_lifecycle-ANY-20260101T120000Z" + + ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + UUID.randomUUID() + ExportFileManager.EXPORT_ARCHIVE_SUFFIX; + assertEquals("20260101T120000Z", ExportFileManager.archiveTimestampFromArchiveFileName(fileName)); + } + @Test public void testListCompletedArchivePaths() throws Exception { ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); - String olderJobId = UUID.randomUUID().toString(); + ExportJob.Id olderJobId = ExportJob.Id.newId(); File olderArchive = fileManager.resolveArchiveFile(scope, "20260101T120000Z", olderJobId); assertTrue(olderArchive.createNewFile()); - assertTrue(olderArchive.setLastModified(1_000L)); - String newerJobId = UUID.randomUUID().toString(); + assertTrue(olderArchive.setLastModified(2_000L)); + ExportJob.Id newerJobId = ExportJob.Id.newId(); File newerArchive = fileManager.resolveArchiveFile(scope, "20260101T120001Z", newerJobId); assertTrue(newerArchive.createNewFile()); - assertTrue(newerArchive.setLastModified(2_000L)); - String tempJobId = UUID.randomUUID().toString(); + assertTrue(newerArchive.setLastModified(1_000L)); + ExportJob.Id tempJobId = ExportJob.Id.newId(); File tempArchive = fileManager.resolveArchiveTempFile(scope, "20260101T120002Z", tempJobId); assertTrue(tempArchive.createNewFile()); @@ -98,7 +115,7 @@ public void testListCompletedArchivePaths() throws Exception { @Test public void testOrphanJobDirRemovedOnStartup() throws Exception { - String jobId = UUID.randomUUID().toString(); + ExportJob.Id jobId = ExportJob.Id.newId(); Path orphanJobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(orphanJobDir); @@ -109,7 +126,7 @@ public void testOrphanJobDirRemovedOnStartup() throws Exception { @Test public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { - String jobId = UUID.randomUUID().toString(); + ExportJob.Id jobId = ExportJob.Id.newId(); Path jobDir = tempDir.toPath().resolve(ExportFileManager.exportJobDirName(jobId)); Files.createDirectories(jobDir); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); @@ -124,7 +141,7 @@ public void testIncompleteExportArtifactsRemovedOnStartup() throws Exception { @Test public void testOrphanJobDirDoesNotDeleteCompletedTar() throws Exception { - String jobId = UUID.randomUUID().toString(); + ExportJob.Id jobId = ExportJob.Id.newId(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); File completedArchive = fileManager.resolveArchiveFile(scope, "20260101T000000Z", jobId); assertTrue(completedArchive.createNewFile()); From 2a49695cba477a4a16985d98d373bd77d2b45ecc Mon Sep 17 00:00:00 2001 From: sarvekshayr Date: Sat, 1 Aug 2026 20:24:38 +0530 Subject: [PATCH 7/7] Use - within component and _ for inter-component --- .../hdds/scm/container/export/ExportFileManager.java | 10 +++++----- .../hadoop/hdds/scm/container/export/ExportScope.java | 2 +- .../scm/container/export/TestExportFileManager.java | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java index f75a6e79ddaf..e0bca340a241 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportFileManager.java @@ -46,17 +46,17 @@ * *

While a job runs, shard text files are written under {@code export_{jobId}/}. The archive is * created only after all shards are written. The export manager writes - * {@code container-ids-{scope}-{timestamp}_job{jobId}.tar.gz.tmp} and atomically renames it to + * {@code container-ids_{scope}_{timestamp}_job{jobId}.tar.gz.tmp} and atomically renames it to * {@code .tar.gz} on close ({@link AtomicFileOutputStream}), so a partial {@code .tar.gz} is * never visible. {@link #lock()} uses {@code in_use.lock} to exclude concurrent writers. * * 

  * {exportDirectory}/
  * ├── in_use.lock
- * ├── container-ids-{scope}-{timestamp}_job{jobId}.tar.gz
- * ├── container-ids-{scope}-{timestamp}_job{jobId}.tar.gz.tmp
+ * ├── container-ids_{scope}_{timestamp}_job{jobId}.tar.gz
+ * ├── container-ids_{scope}_{timestamp}_job{jobId}.tar.gz.tmp
  * └── export_{jobId}/
- *     ├── container-ids-{scope}-{timestamp}-part001.txt
+ *     ├── container-ids_{scope}_{metadataTimestamp}_part001.txt
  *     └── ...
  *
* @@ -129,7 +129,7 @@ void unlock() throws IOException { } File resolveArchiveFile(ExportScope scope, String archiveTimestamp, ExportJob.Id jobId) { - return new File(exportDirectory, String.format("container-ids-%s-%s%s%s%s", + return new File(exportDirectory, String.format("container-ids_%s_%s%s%s%s", scope.getValue(), archiveTimestamp, EXPORT_ARCHIVE_JOB_INFIX, jobId.getValue(), EXPORT_ARCHIVE_SUFFIX)); } diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java index 7c488ab799eb..921fdf7f588f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/export/ExportScope.java @@ -24,7 +24,7 @@ * Container listing filters for an export job. * An export job filters containers by {@link ContainerHealthState}, {@link LifeCycleState} or both. * Example archive name: - * {@code container-ids-health-MISSING_lifecycle-OPEN-20260101T120000Z_job{jobId}.tar.gz} + * {@code container-ids_health-MISSING_lifecycle-OPEN_20260101T120000Z_job{jobId}.tar.gz} */ public final class ExportScope { diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java index 67e3d5f17818..51bb88a56f50 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/export/TestExportFileManager.java @@ -62,7 +62,7 @@ public void testResolveArchiveFile() { ExportJob.Id jobId = ExportJob.Id.newId(); ExportScope scope = ExportScope.of(null, ContainerHealthState.MISSING); File archive = fileManager.resolveArchiveFile(scope, "20260101T120000Z", jobId); - assertTrue(archive.getName().contains("health-MISSING_lifecycle-ANY-20260101T120000Z")); + assertTrue(archive.getName().contains("health-MISSING_lifecycle-ANY_20260101T120000Z")); assertTrue(archive.getName().endsWith(ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId.getValue() + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); } @@ -78,16 +78,16 @@ public void testResolveArchiveTempFile() { @Test public void testJobIdFromArchiveFileName() { String jobId = UUID.randomUUID().toString(); - String fileName = "container-ids-health-MISSING_lifecycle-ANY-20260101T120000Z" + String fileName = "container-ids_health-MISSING_lifecycle-ANY_20260101T120000Z" + ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + jobId + ExportFileManager.EXPORT_ARCHIVE_SUFFIX; assertEquals(ExportJob.Id.of(jobId), ExportFileManager.jobIdFromArchiveFileName(fileName)); - assertNull(ExportFileManager.jobIdFromArchiveFileName("container-ids-health-MISSING_lifecycle-ANY-20260101T120000Z" + assertNull(ExportFileManager.jobIdFromArchiveFileName("container-ids_health-MISSING_lifecycle-ANY_20260101T120000Z" + ExportFileManager.EXPORT_ARCHIVE_SUFFIX)); } @Test public void testArchiveTimestampFromArchiveFileName() { - String fileName = "container-ids-health-MISSING_lifecycle-ANY-20260101T120000Z" + String fileName = "container-ids_health-MISSING_lifecycle-ANY_20260101T120000Z" + ExportFileManager.EXPORT_ARCHIVE_JOB_INFIX + UUID.randomUUID() + ExportFileManager.EXPORT_ARCHIVE_SUFFIX; assertEquals("20260101T120000Z", ExportFileManager.archiveTimestampFromArchiveFileName(fileName)); }